2024-11-21 01:43:48,473 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f 2024-11-21 01:43:48,489 main DEBUG Took 0.013748 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-21 01:43:48,490 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-21 01:43:48,490 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-21 01:43:48,492 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-21 01:43:48,493 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 01:43:48,503 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-21 01:43:48,529 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 01:43:48,531 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 01:43:48,532 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 01:43:48,532 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 01:43:48,533 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 01:43:48,533 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 01:43:48,534 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 01:43:48,535 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 01:43:48,536 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 01:43:48,536 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 01:43:48,538 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 01:43:48,538 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 01:43:48,539 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 01:43:48,539 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 01:43:48,540 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 01:43:48,540 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 01:43:48,541 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 01:43:48,541 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 01:43:48,542 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 01:43:48,542 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 01:43:48,543 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 01:43:48,543 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 01:43:48,544 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 01:43:48,544 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 01:43:48,545 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 01:43:48,546 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-21 01:43:48,547 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 01:43:48,555 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-21 01:43:48,559 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-21 01:43:48,560 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-21 01:43:48,561 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-21 01:43:48,562 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-21 01:43:48,572 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-21 01:43:48,575 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-21 01:43:48,577 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-21 01:43:48,577 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-21 01:43:48,578 main DEBUG createAppenders(={Console}) 2024-11-21 01:43:48,582 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f initialized 2024-11-21 01:43:48,583 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f 2024-11-21 01:43:48,583 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f OK. 2024-11-21 01:43:48,584 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-21 01:43:48,584 main DEBUG OutputStream closed 2024-11-21 01:43:48,584 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-21 01:43:48,585 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-21 01:43:48,585 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4b0d79fc OK 2024-11-21 01:43:48,683 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-21 01:43:48,686 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-21 01:43:48,688 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-21 01:43:48,689 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-21 01:43:48,690 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-21 01:43:48,690 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-21 01:43:48,691 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-21 01:43:48,691 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-21 01:43:48,692 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-21 01:43:48,692 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-21 01:43:48,693 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-21 01:43:48,693 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-21 01:43:48,694 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-21 01:43:48,694 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-21 01:43:48,694 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-21 01:43:48,695 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-21 01:43:48,695 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-21 01:43:48,696 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-21 01:43:48,702 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21 01:43:48,702 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@68b32e3e) with optional ClassLoader: null 2024-11-21 01:43:48,703 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-21 01:43:48,704 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@68b32e3e] started OK. 2024-11-21T01:43:49,087 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0 2024-11-21 01:43:49,091 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-21 01:43:49,091 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21T01:43:49,105 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-21T01:43:49,130 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-21T01:43:49,153 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/cluster_5ec1f663-147e-5567-4614-f2cd450fd939, deleteOnExit=true 2024-11-21T01:43:49,154 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-21T01:43:49,155 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/test.cache.data in system properties and HBase conf 2024-11-21T01:43:49,156 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/hadoop.tmp.dir in system properties and HBase conf 2024-11-21T01:43:49,157 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/hadoop.log.dir in system properties and HBase conf 2024-11-21T01:43:49,158 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-21T01:43:49,158 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-21T01:43:49,159 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-21T01:43:49,305 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-21T01:43:49,457 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T01:43:49,462 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-21T01:43:49,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-21T01:43:49,469 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-21T01:43:49,470 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-21T01:43:49,471 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-21T01:43:49,471 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-21T01:43:49,472 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-21T01:43:49,473 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-21T01:43:49,473 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-21T01:43:49,474 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/nfs.dump.dir in system properties and HBase conf 2024-11-21T01:43:49,474 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/java.io.tmpdir in system properties and HBase conf 2024-11-21T01:43:49,475 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-21T01:43:49,475 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-21T01:43:49,476 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-21T01:43:50,556 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-21T01:43:50,650 INFO [Time-limited test {}] log.Log(170): Logging initialized @3181ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-21T01:43:50,751 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T01:43:50,843 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T01:43:50,871 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T01:43:50,871 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T01:43:50,872 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-21T01:43:50,885 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T01:43:50,889 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/hadoop.log.dir/,AVAILABLE} 2024-11-21T01:43:50,890 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T01:43:51,142 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2341a969{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/java.io.tmpdir/jetty-localhost-42643-hadoop-hdfs-3_4_1-tests_jar-_-any-5146103344412932997/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-21T01:43:51,151 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:42643} 2024-11-21T01:43:51,151 INFO [Time-limited test {}] server.Server(415): Started @3683ms 2024-11-21T01:43:51,668 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T01:43:51,677 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T01:43:51,680 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T01:43:51,680 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T01:43:51,680 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-21T01:43:51,685 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4828e07b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/hadoop.log.dir/,AVAILABLE} 2024-11-21T01:43:51,686 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@54a7b62e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T01:43:51,864 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@72835e42{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/java.io.tmpdir/jetty-localhost-43357-hadoop-hdfs-3_4_1-tests_jar-_-any-3844266685634129069/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T01:43:51,865 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2098bdc7{HTTP/1.1, (http/1.1)}{localhost:43357} 2024-11-21T01:43:51,866 INFO [Time-limited test {}] server.Server(415): Started @4398ms 2024-11-21T01:43:51,928 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-21T01:43:52,463 WARN [Thread-71 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/cluster_5ec1f663-147e-5567-4614-f2cd450fd939/data/data1/current/BP-1273700343-172.17.0.3-1732153430280/current, will proceed with Du for space computation calculation, 2024-11-21T01:43:52,490 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/cluster_5ec1f663-147e-5567-4614-f2cd450fd939/data/data2/current/BP-1273700343-172.17.0.3-1732153430280/current, will proceed with Du for space computation calculation, 2024-11-21T01:43:52,530 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-21T01:43:52,597 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb1eecc1e7f21649d with lease ID 0xc5d31c0d2d033ada: Processing first storage report for DS-40acecc4-503d-4e61-be4c-078df12719bd from datanode DatanodeRegistration(127.0.0.1:45349, datanodeUuid=1349e6fd-a662-4842-aff7-117c872bca27, infoPort=34705, infoSecurePort=0, ipcPort=43391, storageInfo=lv=-57;cid=testClusterID;nsid=13038040;c=1732153430280) 2024-11-21T01:43:52,598 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb1eecc1e7f21649d with lease ID 0xc5d31c0d2d033ada: from storage DS-40acecc4-503d-4e61-be4c-078df12719bd node DatanodeRegistration(127.0.0.1:45349, datanodeUuid=1349e6fd-a662-4842-aff7-117c872bca27, infoPort=34705, infoSecurePort=0, ipcPort=43391, storageInfo=lv=-57;cid=testClusterID;nsid=13038040;c=1732153430280), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-21T01:43:52,599 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb1eecc1e7f21649d with lease ID 0xc5d31c0d2d033ada: Processing first storage report for DS-346ceba3-913d-4736-bd26-a6a0999cc1cc from datanode DatanodeRegistration(127.0.0.1:45349, datanodeUuid=1349e6fd-a662-4842-aff7-117c872bca27, infoPort=34705, infoSecurePort=0, ipcPort=43391, storageInfo=lv=-57;cid=testClusterID;nsid=13038040;c=1732153430280) 2024-11-21T01:43:52,599 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb1eecc1e7f21649d with lease ID 0xc5d31c0d2d033ada: from storage DS-346ceba3-913d-4736-bd26-a6a0999cc1cc node DatanodeRegistration(127.0.0.1:45349, datanodeUuid=1349e6fd-a662-4842-aff7-117c872bca27, infoPort=34705, infoSecurePort=0, ipcPort=43391, storageInfo=lv=-57;cid=testClusterID;nsid=13038040;c=1732153430280), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-21T01:43:52,624 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0 2024-11-21T01:43:52,734 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/cluster_5ec1f663-147e-5567-4614-f2cd450fd939/zookeeper_0, clientPort=57444, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/cluster_5ec1f663-147e-5567-4614-f2cd450fd939/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/cluster_5ec1f663-147e-5567-4614-f2cd450fd939/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-21T01:43:52,750 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57444 2024-11-21T01:43:52,760 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T01:43:52,763 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T01:43:53,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741825_1001 (size=7) 2024-11-21T01:43:53,422 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d with version=8 2024-11-21T01:43:53,423 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/hbase-staging 2024-11-21T01:43:53,551 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-21T01:43:53,834 INFO [Time-limited test {}] client.ConnectionUtils(128): master/b93ada835ef8:0 server-side Connection retries=45 2024-11-21T01:43:53,845 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T01:43:53,846 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T01:43:53,851 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T01:43:53,851 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T01:43:53,851 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T01:43:53,993 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-21T01:43:54,056 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-21T01:43:54,068 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-21T01:43:54,074 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T01:43:54,111 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3726 (auto-detected) 2024-11-21T01:43:54,113 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-21T01:43:54,139 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36607 2024-11-21T01:43:54,162 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36607 connecting to ZooKeeper ensemble=127.0.0.1:57444 2024-11-21T01:43:54,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:366070x0, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T01:43:54,198 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36607-0x10112f6cf300000 connected 2024-11-21T01:43:54,225 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T01:43:54,227 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T01:43:54,238 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36607-0x10112f6cf300000, quorum=127.0.0.1:57444, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T01:43:54,243 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d, hbase.cluster.distributed=false 2024-11-21T01:43:54,270 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36607-0x10112f6cf300000, quorum=127.0.0.1:57444, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T01:43:54,275 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36607 2024-11-21T01:43:54,275 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36607 2024-11-21T01:43:54,276 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36607 2024-11-21T01:43:54,276 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36607 2024-11-21T01:43:54,277 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36607 2024-11-21T01:43:54,402 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b93ada835ef8:0 server-side Connection retries=45 2024-11-21T01:43:54,404 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T01:43:54,404 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T01:43:54,405 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T01:43:54,405 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T01:43:54,405 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T01:43:54,408 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T01:43:54,410 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T01:43:54,411 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41129 2024-11-21T01:43:54,413 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41129 connecting to ZooKeeper ensemble=127.0.0.1:57444 2024-11-21T01:43:54,415 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T01:43:54,418 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T01:43:54,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:411290x0, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T01:43:54,434 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:411290x0, quorum=127.0.0.1:57444, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T01:43:54,435 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41129-0x10112f6cf300001 connected 2024-11-21T01:43:54,438 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T01:43:54,446 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T01:43:54,449 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41129-0x10112f6cf300001, quorum=127.0.0.1:57444, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T01:43:54,454 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41129-0x10112f6cf300001, quorum=127.0.0.1:57444, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T01:43:54,457 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41129 2024-11-21T01:43:54,458 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41129 2024-11-21T01:43:54,459 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41129 2024-11-21T01:43:54,462 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41129 2024-11-21T01:43:54,465 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41129 2024-11-21T01:43:54,487 DEBUG [M:0;b93ada835ef8:36607 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;b93ada835ef8:36607 2024-11-21T01:43:54,488 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/b93ada835ef8,36607,1732153433607 2024-11-21T01:43:54,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36607-0x10112f6cf300000, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T01:43:54,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41129-0x10112f6cf300001, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T01:43:54,498 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36607-0x10112f6cf300000, quorum=127.0.0.1:57444, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b93ada835ef8,36607,1732153433607 2024-11-21T01:43:54,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41129-0x10112f6cf300001, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T01:43:54,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36607-0x10112f6cf300000, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:43:54,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41129-0x10112f6cf300001, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:43:54,526 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36607-0x10112f6cf300000, quorum=127.0.0.1:57444, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-21T01:43:54,527 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b93ada835ef8,36607,1732153433607 from backup master directory 2024-11-21T01:43:54,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36607-0x10112f6cf300000, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b93ada835ef8,36607,1732153433607 2024-11-21T01:43:54,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41129-0x10112f6cf300001, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T01:43:54,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36607-0x10112f6cf300000, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T01:43:54,531 WARN [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T01:43:54,531 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b93ada835ef8,36607,1732153433607 2024-11-21T01:43:54,534 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-21T01:43:54,535 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-21T01:43:54,592 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/hbase.id] with ID: 809ac628-2316-4d75-a0fd-5a443f7d2388 2024-11-21T01:43:54,592 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/.tmp/hbase.id 2024-11-21T01:43:54,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741826_1002 (size=42) 2024-11-21T01:43:55,005 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/.tmp/hbase.id]:[hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/hbase.id] 2024-11-21T01:43:55,051 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T01:43:55,057 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-21T01:43:55,077 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-21T01:43:55,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41129-0x10112f6cf300001, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:43:55,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36607-0x10112f6cf300000, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:43:55,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741827_1003 (size=196) 2024-11-21T01:43:55,114 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T01:43:55,117 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-21T01:43:55,135 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:43:55,140 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T01:43:55,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741828_1004 (size=1189) 2024-11-21T01:43:55,216 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/MasterData/data/master/store 2024-11-21T01:43:55,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741829_1005 (size=34) 2024-11-21T01:43:55,653 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-21T01:43:55,656 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:43:55,658 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T01:43:55,658 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T01:43:55,658 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T01:43:55,660 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T01:43:55,660 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T01:43:55,660 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T01:43:55,662 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732153435657Disabling compacts and flushes for region at 1732153435657Disabling writes for close at 1732153435660 (+3 ms)Writing region close event to WAL at 1732153435660Closed at 1732153435660 2024-11-21T01:43:55,665 WARN [master/b93ada835ef8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/MasterData/data/master/store/.initializing 2024-11-21T01:43:55,665 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/MasterData/WALs/b93ada835ef8,36607,1732153433607 2024-11-21T01:43:55,674 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T01:43:55,688 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b93ada835ef8%2C36607%2C1732153433607, suffix=, logDir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/MasterData/WALs/b93ada835ef8,36607,1732153433607, archiveDir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/MasterData/oldWALs, maxLogs=10 2024-11-21T01:43:55,711 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/MasterData/WALs/b93ada835ef8,36607,1732153433607/b93ada835ef8%2C36607%2C1732153433607.1732153435692, exclude list is [], retry=0 2024-11-21T01:43:55,731 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45349,DS-40acecc4-503d-4e61-be4c-078df12719bd,DISK] 2024-11-21T01:43:55,735 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-21T01:43:55,780 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/MasterData/WALs/b93ada835ef8,36607,1732153433607/b93ada835ef8%2C36607%2C1732153433607.1732153435692 2024-11-21T01:43:55,782 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34705:34705)] 2024-11-21T01:43:55,782 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-21T01:43:55,783 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:43:55,786 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:43:55,788 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:43:55,833 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:43:55,866 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-21T01:43:55,870 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:43:55,873 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T01:43:55,874 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:43:55,878 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-21T01:43:55,878 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:43:55,879 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:43:55,879 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:43:55,882 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-21T01:43:55,883 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:43:55,884 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:43:55,884 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:43:55,887 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-21T01:43:55,887 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:43:55,889 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:43:55,889 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:43:55,893 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:43:55,895 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:43:55,901 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:43:55,902 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:43:55,907 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T01:43:55,914 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:43:55,920 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T01:43:55,921 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68005569, jitterRate=0.013361945748329163}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T01:43:55,927 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732153435802Initializing all the Stores at 1732153435805 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732153435806 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732153435806Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732153435807 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732153435807Cleaning up temporary data from old regions at 1732153435902 (+95 ms)Region opened successfully at 1732153435927 (+25 ms) 2024-11-21T01:43:55,929 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-21T01:43:55,970 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eed2c28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b93ada835ef8/172.17.0.3:0 2024-11-21T01:43:56,015 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-21T01:43:56,031 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-21T01:43:56,031 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-21T01:43:56,035 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T01:43:56,037 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-21T01:43:56,043 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-21T01:43:56,043 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-21T01:43:56,079 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-21T01:43:56,091 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36607-0x10112f6cf300000, quorum=127.0.0.1:57444, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-21T01:43:56,093 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-21T01:43:56,096 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-21T01:43:56,098 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36607-0x10112f6cf300000, quorum=127.0.0.1:57444, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-21T01:43:56,099 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-21T01:43:56,102 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-21T01:43:56,108 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36607-0x10112f6cf300000, quorum=127.0.0.1:57444, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-21T01:43:56,112 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-21T01:43:56,121 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36607-0x10112f6cf300000, quorum=127.0.0.1:57444, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-21T01:43:56,127 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-21T01:43:56,154 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36607-0x10112f6cf300000, quorum=127.0.0.1:57444, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-21T01:43:56,156 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-21T01:43:56,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41129-0x10112f6cf300001, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T01:43:56,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41129-0x10112f6cf300001, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:43:56,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36607-0x10112f6cf300000, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T01:43:56,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36607-0x10112f6cf300000, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:43:56,172 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=b93ada835ef8,36607,1732153433607, sessionid=0x10112f6cf300000, setting cluster-up flag (Was=false) 2024-11-21T01:43:56,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36607-0x10112f6cf300000, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:43:56,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41129-0x10112f6cf300001, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:43:56,199 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-21T01:43:56,203 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b93ada835ef8,36607,1732153433607 2024-11-21T01:43:56,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36607-0x10112f6cf300000, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:43:56,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41129-0x10112f6cf300001, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:43:56,218 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-21T01:43:56,220 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b93ada835ef8,36607,1732153433607 2024-11-21T01:43:56,229 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-21T01:43:56,277 INFO [RS:0;b93ada835ef8:41129 {}] regionserver.HRegionServer(746): ClusterId : 809ac628-2316-4d75-a0fd-5a443f7d2388 2024-11-21T01:43:56,280 DEBUG [RS:0;b93ada835ef8:41129 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T01:43:56,288 DEBUG [RS:0;b93ada835ef8:41129 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T01:43:56,289 DEBUG [RS:0;b93ada835ef8:41129 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T01:43:56,295 DEBUG [RS:0;b93ada835ef8:41129 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T01:43:56,296 DEBUG [RS:0;b93ada835ef8:41129 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18dd435c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b93ada835ef8/172.17.0.3:0 2024-11-21T01:43:56,317 DEBUG [RS:0;b93ada835ef8:41129 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;b93ada835ef8:41129 2024-11-21T01:43:56,322 INFO [RS:0;b93ada835ef8:41129 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T01:43:56,322 INFO [RS:0;b93ada835ef8:41129 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T01:43:56,322 DEBUG [RS:0;b93ada835ef8:41129 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T01:43:56,325 INFO [RS:0;b93ada835ef8:41129 {}] regionserver.HRegionServer(2659): reportForDuty to master=b93ada835ef8,36607,1732153433607 with port=41129, startcode=1732153434359 2024-11-21T01:43:56,333 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T01:43:56,340 DEBUG [RS:0;b93ada835ef8:41129 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T01:43:56,346 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-21T01:43:56,354 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-21T01:43:56,361 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b93ada835ef8,36607,1732153433607 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-21T01:43:56,377 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b93ada835ef8:0, corePoolSize=5, maxPoolSize=5 2024-11-21T01:43:56,377 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b93ada835ef8:0, corePoolSize=5, maxPoolSize=5 2024-11-21T01:43:56,377 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b93ada835ef8:0, corePoolSize=5, maxPoolSize=5 2024-11-21T01:43:56,378 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b93ada835ef8:0, corePoolSize=5, maxPoolSize=5 2024-11-21T01:43:56,378 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b93ada835ef8:0, corePoolSize=10, maxPoolSize=10 2024-11-21T01:43:56,378 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:43:56,378 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b93ada835ef8:0, corePoolSize=2, maxPoolSize=2 2024-11-21T01:43:56,379 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:43:56,387 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58879, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T01:43:56,392 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T01:43:56,392 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-21T01:43:56,395 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36607 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:43:56,402 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:43:56,403 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-21T01:43:56,418 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732153466418 2024-11-21T01:43:56,420 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-21T01:43:56,421 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-21T01:43:56,426 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-21T01:43:56,426 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-21T01:43:56,430 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-21T01:43:56,431 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-21T01:43:56,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741831_1007 (size=1321) 2024-11-21T01:43:56,433 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T01:43:56,437 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-21T01:43:56,439 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-21T01:43:56,439 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-21T01:43:56,440 DEBUG [RS:0;b93ada835ef8:41129 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-21T01:43:56,441 WARN [RS:0;b93ada835ef8:41129 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-21T01:43:56,442 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-21T01:43:56,443 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-21T01:43:56,445 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b93ada835ef8:0:becomeActiveMaster-HFileCleaner.large.0-1732153436444,5,FailOnTimeoutGroup] 2024-11-21T01:43:56,446 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b93ada835ef8:0:becomeActiveMaster-HFileCleaner.small.0-1732153436445,5,FailOnTimeoutGroup] 2024-11-21T01:43:56,446 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T01:43:56,446 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-21T01:43:56,447 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-21T01:43:56,448 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-21T01:43:56,543 INFO [RS:0;b93ada835ef8:41129 {}] regionserver.HRegionServer(2659): reportForDuty to master=b93ada835ef8,36607,1732153433607 with port=41129, startcode=1732153434359 2024-11-21T01:43:56,545 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36607 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b93ada835ef8,41129,1732153434359 2024-11-21T01:43:56,548 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36607 {}] master.ServerManager(517): Registering regionserver=b93ada835ef8,41129,1732153434359 2024-11-21T01:43:56,558 DEBUG [RS:0;b93ada835ef8:41129 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d 2024-11-21T01:43:56,558 DEBUG [RS:0;b93ada835ef8:41129 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37693 2024-11-21T01:43:56,558 DEBUG [RS:0;b93ada835ef8:41129 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T01:43:56,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36607-0x10112f6cf300000, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T01:43:56,564 DEBUG [RS:0;b93ada835ef8:41129 {}] zookeeper.ZKUtil(111): regionserver:41129-0x10112f6cf300001, quorum=127.0.0.1:57444, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b93ada835ef8,41129,1732153434359 2024-11-21T01:43:56,565 WARN [RS:0;b93ada835ef8:41129 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T01:43:56,565 INFO [RS:0;b93ada835ef8:41129 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T01:43:56,565 DEBUG [RS:0;b93ada835ef8:41129 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/WALs/b93ada835ef8,41129,1732153434359 2024-11-21T01:43:56,567 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b93ada835ef8,41129,1732153434359] 2024-11-21T01:43:56,600 INFO [RS:0;b93ada835ef8:41129 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T01:43:56,614 INFO [RS:0;b93ada835ef8:41129 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T01:43:56,620 INFO [RS:0;b93ada835ef8:41129 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T01:43:56,620 INFO [RS:0;b93ada835ef8:41129 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T01:43:56,621 INFO [RS:0;b93ada835ef8:41129 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T01:43:56,627 INFO [RS:0;b93ada835ef8:41129 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T01:43:56,629 INFO [RS:0;b93ada835ef8:41129 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T01:43:56,629 DEBUG [RS:0;b93ada835ef8:41129 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:43:56,630 DEBUG [RS:0;b93ada835ef8:41129 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:43:56,630 DEBUG [RS:0;b93ada835ef8:41129 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:43:56,630 DEBUG [RS:0;b93ada835ef8:41129 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:43:56,630 DEBUG [RS:0;b93ada835ef8:41129 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:43:56,631 DEBUG [RS:0;b93ada835ef8:41129 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b93ada835ef8:0, corePoolSize=2, maxPoolSize=2 2024-11-21T01:43:56,631 DEBUG [RS:0;b93ada835ef8:41129 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:43:56,631 DEBUG [RS:0;b93ada835ef8:41129 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:43:56,631 DEBUG [RS:0;b93ada835ef8:41129 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:43:56,632 DEBUG [RS:0;b93ada835ef8:41129 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:43:56,632 DEBUG [RS:0;b93ada835ef8:41129 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:43:56,632 DEBUG [RS:0;b93ada835ef8:41129 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:43:56,632 DEBUG [RS:0;b93ada835ef8:41129 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b93ada835ef8:0, corePoolSize=3, maxPoolSize=3 2024-11-21T01:43:56,633 DEBUG [RS:0;b93ada835ef8:41129 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0, corePoolSize=3, maxPoolSize=3 2024-11-21T01:43:56,634 INFO [RS:0;b93ada835ef8:41129 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T01:43:56,634 INFO [RS:0;b93ada835ef8:41129 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T01:43:56,634 INFO [RS:0;b93ada835ef8:41129 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T01:43:56,635 INFO [RS:0;b93ada835ef8:41129 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T01:43:56,635 INFO [RS:0;b93ada835ef8:41129 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T01:43:56,635 INFO [RS:0;b93ada835ef8:41129 {}] hbase.ChoreService(168): Chore ScheduledChore name=b93ada835ef8,41129,1732153434359-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T01:43:56,660 INFO [RS:0;b93ada835ef8:41129 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T01:43:56,663 INFO [RS:0;b93ada835ef8:41129 {}] hbase.ChoreService(168): Chore ScheduledChore name=b93ada835ef8,41129,1732153434359-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T01:43:56,663 INFO [RS:0;b93ada835ef8:41129 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T01:43:56,663 INFO [RS:0;b93ada835ef8:41129 {}] regionserver.Replication(171): b93ada835ef8,41129,1732153434359 started 2024-11-21T01:43:56,688 INFO [RS:0;b93ada835ef8:41129 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T01:43:56,688 INFO [RS:0;b93ada835ef8:41129 {}] regionserver.HRegionServer(1482): Serving as b93ada835ef8,41129,1732153434359, RpcServer on b93ada835ef8/172.17.0.3:41129, sessionid=0x10112f6cf300001 2024-11-21T01:43:56,689 DEBUG [RS:0;b93ada835ef8:41129 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T01:43:56,689 DEBUG [RS:0;b93ada835ef8:41129 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b93ada835ef8,41129,1732153434359 2024-11-21T01:43:56,689 DEBUG [RS:0;b93ada835ef8:41129 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b93ada835ef8,41129,1732153434359' 2024-11-21T01:43:56,690 DEBUG [RS:0;b93ada835ef8:41129 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T01:43:56,691 DEBUG [RS:0;b93ada835ef8:41129 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T01:43:56,692 DEBUG [RS:0;b93ada835ef8:41129 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T01:43:56,692 DEBUG [RS:0;b93ada835ef8:41129 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T01:43:56,692 DEBUG [RS:0;b93ada835ef8:41129 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b93ada835ef8,41129,1732153434359 2024-11-21T01:43:56,692 DEBUG [RS:0;b93ada835ef8:41129 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b93ada835ef8,41129,1732153434359' 2024-11-21T01:43:56,692 DEBUG [RS:0;b93ada835ef8:41129 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T01:43:56,693 DEBUG [RS:0;b93ada835ef8:41129 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T01:43:56,694 DEBUG [RS:0;b93ada835ef8:41129 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T01:43:56,694 INFO [RS:0;b93ada835ef8:41129 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T01:43:56,694 INFO [RS:0;b93ada835ef8:41129 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T01:43:56,801 INFO [RS:0;b93ada835ef8:41129 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T01:43:56,806 INFO [RS:0;b93ada835ef8:41129 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b93ada835ef8%2C41129%2C1732153434359, suffix=, logDir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/WALs/b93ada835ef8,41129,1732153434359, archiveDir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/oldWALs, maxLogs=32 2024-11-21T01:43:56,830 DEBUG [RS:0;b93ada835ef8:41129 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/WALs/b93ada835ef8,41129,1732153434359/b93ada835ef8%2C41129%2C1732153434359.1732153436809, exclude list is [], retry=0 2024-11-21T01:43:56,834 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-21T01:43:56,835 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d 2024-11-21T01:43:56,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45349,DS-40acecc4-503d-4e61-be4c-078df12719bd,DISK] 2024-11-21T01:43:56,844 INFO [RS:0;b93ada835ef8:41129 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/WALs/b93ada835ef8,41129,1732153434359/b93ada835ef8%2C41129%2C1732153434359.1732153436809 2024-11-21T01:43:56,847 DEBUG [RS:0;b93ada835ef8:41129 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34705:34705)] 2024-11-21T01:43:56,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741833_1009 (size=32) 2024-11-21T01:43:57,257 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:43:57,260 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T01:43:57,263 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T01:43:57,263 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:43:57,264 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T01:43:57,264 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T01:43:57,267 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T01:43:57,267 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:43:57,268 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T01:43:57,269 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T01:43:57,271 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T01:43:57,271 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:43:57,272 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T01:43:57,272 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T01:43:57,275 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T01:43:57,275 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:43:57,276 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T01:43:57,276 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T01:43:57,278 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/hbase/meta/1588230740 2024-11-21T01:43:57,280 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/hbase/meta/1588230740 2024-11-21T01:43:57,284 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T01:43:57,284 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T01:43:57,285 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T01:43:57,287 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T01:43:57,292 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T01:43:57,293 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66375017, jitterRate=-0.010935172438621521}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T01:43:57,296 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732153437257Initializing all the Stores at 1732153437259 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732153437259Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732153437259Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732153437259Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732153437260 (+1 ms)Cleaning up temporary data from old regions at 1732153437284 (+24 ms)Region opened successfully at 1732153437296 (+12 ms) 2024-11-21T01:43:57,296 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T01:43:57,296 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T01:43:57,296 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T01:43:57,297 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T01:43:57,297 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T01:43:57,298 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T01:43:57,298 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732153437296Disabling compacts and flushes for region at 1732153437296Disabling writes for close at 1732153437297 (+1 ms)Writing region close event to WAL at 1732153437298 (+1 ms)Closed at 1732153437298 2024-11-21T01:43:57,302 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T01:43:57,302 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-21T01:43:57,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T01:43:57,320 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T01:43:57,324 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-21T01:43:57,478 DEBUG [b93ada835ef8:36607 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-21T01:43:57,494 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b93ada835ef8,41129,1732153434359 2024-11-21T01:43:57,501 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b93ada835ef8,41129,1732153434359, state=OPENING 2024-11-21T01:43:57,506 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-21T01:43:57,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41129-0x10112f6cf300001, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:43:57,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36607-0x10112f6cf300000, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:43:57,510 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T01:43:57,512 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T01:43:57,512 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T01:43:57,514 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=b93ada835ef8,41129,1732153434359}] 2024-11-21T01:43:57,697 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T01:43:57,701 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47433, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T01:43:57,714 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T01:43:57,714 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T01:43:57,715 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T01:43:57,720 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b93ada835ef8%2C41129%2C1732153434359.meta, suffix=.meta, logDir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/WALs/b93ada835ef8,41129,1732153434359, archiveDir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/oldWALs, maxLogs=32 2024-11-21T01:43:57,745 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/WALs/b93ada835ef8,41129,1732153434359/b93ada835ef8%2C41129%2C1732153434359.meta.1732153437723.meta, exclude list is [], retry=0 2024-11-21T01:43:57,751 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45349,DS-40acecc4-503d-4e61-be4c-078df12719bd,DISK] 2024-11-21T01:43:57,754 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/WALs/b93ada835ef8,41129,1732153434359/b93ada835ef8%2C41129%2C1732153434359.meta.1732153437723.meta 2024-11-21T01:43:57,757 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34705:34705)] 2024-11-21T01:43:57,757 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T01:43:57,760 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T01:43:57,764 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T01:43:57,771 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T01:43:57,776 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T01:43:57,777 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:43:57,777 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T01:43:57,777 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T01:43:57,781 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T01:43:57,784 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T01:43:57,784 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:43:57,785 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T01:43:57,785 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T01:43:57,787 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T01:43:57,787 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:43:57,789 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T01:43:57,790 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T01:43:57,792 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T01:43:57,792 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:43:57,793 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T01:43:57,793 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T01:43:57,795 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T01:43:57,795 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:43:57,796 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T01:43:57,796 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T01:43:57,800 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/hbase/meta/1588230740 2024-11-21T01:43:57,803 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/hbase/meta/1588230740 2024-11-21T01:43:57,806 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T01:43:57,806 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T01:43:57,807 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T01:43:57,811 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T01:43:57,813 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68123406, jitterRate=0.015117853879928589}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T01:43:57,813 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T01:43:57,816 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732153437778Writing region info on filesystem at 1732153437778Initializing all the Stores at 1732153437781 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732153437781Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732153437781Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732153437781Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732153437781Cleaning up temporary data from old regions at 1732153437806 (+25 ms)Running coprocessor post-open hooks at 1732153437813 (+7 ms)Region opened successfully at 1732153437815 (+2 ms) 2024-11-21T01:43:57,825 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732153437685 2024-11-21T01:43:57,839 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T01:43:57,840 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T01:43:57,842 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=b93ada835ef8,41129,1732153434359 2024-11-21T01:43:57,844 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b93ada835ef8,41129,1732153434359, state=OPEN 2024-11-21T01:43:57,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41129-0x10112f6cf300001, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T01:43:57,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36607-0x10112f6cf300000, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T01:43:57,851 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T01:43:57,851 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T01:43:57,852 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:43:57,857 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-21T01:43:57,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b93ada835ef8,41129,1732153434359 in 338 msec 2024-11-21T01:43:57,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-21T01:43:57,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 550 msec 2024-11-21T01:43:57,868 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T01:43:57,868 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-21T01:43:57,895 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:43:57,897 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:43:57,927 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:43:57,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37855, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:43:57,968 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.7000 sec 2024-11-21T01:43:57,968 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732153437968, completionTime=-1 2024-11-21T01:43:57,972 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-21T01:43:57,972 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T01:43:58,010 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-21T01:43:58,010 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732153498010 2024-11-21T01:43:58,010 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732153558010 2024-11-21T01:43:58,010 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 38 msec 2024-11-21T01:43:58,013 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b93ada835ef8,36607,1732153433607-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T01:43:58,014 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b93ada835ef8,36607,1732153433607-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T01:43:58,014 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b93ada835ef8,36607,1732153433607-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T01:43:58,016 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b93ada835ef8:36607, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T01:43:58,016 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-21T01:43:58,017 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-21T01:43:58,024 DEBUG [master/b93ada835ef8:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-21T01:43:58,049 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.518sec 2024-11-21T01:43:58,050 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-21T01:43:58,051 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-21T01:43:58,052 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-21T01:43:58,053 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-21T01:43:58,053 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-21T01:43:58,054 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b93ada835ef8,36607,1732153433607-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T01:43:58,054 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b93ada835ef8,36607,1732153433607-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-21T01:43:58,063 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-21T01:43:58,064 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-21T01:43:58,065 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b93ada835ef8,36607,1732153433607-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T01:43:58,090 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61162baf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:43:58,093 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-21T01:43:58,093 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-21T01:43:58,097 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:43:58,100 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:43:58,110 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:43:58,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:43:58,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:43:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@301f7ee4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:43:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:43:58,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:43:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:43:58,124 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52518, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:43:58,128 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c73573a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:43:58,129 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:43:58,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:43:58,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:43:58,140 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58102, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:43:58,163 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=b93ada835ef8,36607,1732153433607 2024-11-21T01:43:58,182 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=481, ProcessCount=11, AvailableMemoryMB=262 2024-11-21T01:43:58,193 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T01:43:58,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is b93ada835ef8,36607,1732153433607 2024-11-21T01:43:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@b1db67 2024-11-21T01:43:58,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T01:43:58,206 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52532, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T01:43:58,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T01:43:58,214 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T01:43:58,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T01:43:58,227 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T01:43:58,230 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-21T01:43:58,230 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:43:58,233 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T01:43:58,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T01:43:58,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741835_1011 (size=1061) 2024-11-21T01:43:58,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T01:43:58,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T01:43:58,653 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d 2024-11-21T01:43:58,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741836_1012 (size=53) 2024-11-21T01:43:58,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T01:43:59,068 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:43:59,068 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 06db9be3c4e6682aa8966442e1ef70cc, disabling compactions & flushes 2024-11-21T01:43:59,068 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:43:59,069 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:43:59,069 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. after waiting 0 ms 2024-11-21T01:43:59,069 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:43:59,069 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:43:59,069 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 06db9be3c4e6682aa8966442e1ef70cc: Waiting for close lock at 1732153439068Disabling compacts and flushes for region at 1732153439068Disabling writes for close at 1732153439069 (+1 ms)Writing region close event to WAL at 1732153439069Closed at 1732153439069 2024-11-21T01:43:59,072 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T01:43:59,080 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732153439072"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732153439072"}]},"ts":"1732153439072"} 2024-11-21T01:43:59,086 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T01:43:59,089 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T01:43:59,092 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732153439089"}]},"ts":"1732153439089"} 2024-11-21T01:43:59,099 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T01:43:59,101 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=06db9be3c4e6682aa8966442e1ef70cc, ASSIGN}] 2024-11-21T01:43:59,106 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=06db9be3c4e6682aa8966442e1ef70cc, ASSIGN 2024-11-21T01:43:59,108 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=06db9be3c4e6682aa8966442e1ef70cc, ASSIGN; state=OFFLINE, location=b93ada835ef8,41129,1732153434359; forceNewPlan=false, retain=false 2024-11-21T01:43:59,261 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=06db9be3c4e6682aa8966442e1ef70cc, regionState=OPENING, regionLocation=b93ada835ef8,41129,1732153434359 2024-11-21T01:43:59,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=06db9be3c4e6682aa8966442e1ef70cc, ASSIGN because future has completed 2024-11-21T01:43:59,268 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359}] 2024-11-21T01:43:59,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T01:43:59,441 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:43:59,441 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} 2024-11-21T01:43:59,442 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:43:59,442 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:43:59,443 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:43:59,443 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:43:59,446 INFO [StoreOpener-06db9be3c4e6682aa8966442e1ef70cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:43:59,451 INFO [StoreOpener-06db9be3c4e6682aa8966442e1ef70cc-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T01:43:59,451 INFO [StoreOpener-06db9be3c4e6682aa8966442e1ef70cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06db9be3c4e6682aa8966442e1ef70cc columnFamilyName A 2024-11-21T01:43:59,451 DEBUG [StoreOpener-06db9be3c4e6682aa8966442e1ef70cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:43:59,452 INFO [StoreOpener-06db9be3c4e6682aa8966442e1ef70cc-1 {}] regionserver.HStore(327): Store=06db9be3c4e6682aa8966442e1ef70cc/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:43:59,453 INFO [StoreOpener-06db9be3c4e6682aa8966442e1ef70cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:43:59,457 INFO [StoreOpener-06db9be3c4e6682aa8966442e1ef70cc-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T01:43:59,458 INFO [StoreOpener-06db9be3c4e6682aa8966442e1ef70cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06db9be3c4e6682aa8966442e1ef70cc columnFamilyName B 2024-11-21T01:43:59,458 DEBUG [StoreOpener-06db9be3c4e6682aa8966442e1ef70cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:43:59,459 INFO [StoreOpener-06db9be3c4e6682aa8966442e1ef70cc-1 {}] regionserver.HStore(327): Store=06db9be3c4e6682aa8966442e1ef70cc/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:43:59,460 INFO [StoreOpener-06db9be3c4e6682aa8966442e1ef70cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:43:59,464 INFO [StoreOpener-06db9be3c4e6682aa8966442e1ef70cc-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T01:43:59,465 INFO [StoreOpener-06db9be3c4e6682aa8966442e1ef70cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06db9be3c4e6682aa8966442e1ef70cc columnFamilyName C 2024-11-21T01:43:59,465 DEBUG [StoreOpener-06db9be3c4e6682aa8966442e1ef70cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:43:59,469 INFO [StoreOpener-06db9be3c4e6682aa8966442e1ef70cc-1 {}] regionserver.HStore(327): Store=06db9be3c4e6682aa8966442e1ef70cc/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:43:59,470 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:43:59,470 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:43:59,472 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:43:59,473 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:43:59,476 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:43:59,476 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:43:59,477 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T01:43:59,480 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:43:59,492 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T01:43:59,493 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 06db9be3c4e6682aa8966442e1ef70cc; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60457447, jitterRate=-0.0991138368844986}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T01:43:59,494 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:43:59,495 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 06db9be3c4e6682aa8966442e1ef70cc: Running coprocessor pre-open hook at 1732153439443Writing region info on filesystem at 1732153439443Initializing all the Stores at 1732153439445 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732153439445Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732153439446 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732153439446Cleaning up temporary data from old regions at 1732153439476 (+30 ms)Running coprocessor post-open hooks at 1732153439494 (+18 ms)Region opened successfully at 1732153439495 (+1 ms) 2024-11-21T01:43:59,498 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., pid=6, masterSystemTime=1732153439426 2024-11-21T01:43:59,505 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:43:59,505 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:43:59,507 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=06db9be3c4e6682aa8966442e1ef70cc, regionState=OPEN, openSeqNum=2, regionLocation=b93ada835ef8,41129,1732153434359 2024-11-21T01:43:59,511 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 because future has completed 2024-11-21T01:43:59,522 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-21T01:43:59,523 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 in 247 msec 2024-11-21T01:43:59,527 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-21T01:43:59,527 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=06db9be3c4e6682aa8966442e1ef70cc, ASSIGN in 420 msec 2024-11-21T01:43:59,532 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T01:43:59,533 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732153439532"}]},"ts":"1732153439532"} 2024-11-21T01:43:59,542 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T01:43:59,544 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T01:43:59,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.3270 sec 2024-11-21T01:44:00,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T01:44:00,411 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:44:00,422 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@652f7cc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:00,422 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:44:00,422 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:44:00,425 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:44:00,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:44:00,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:44:00,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72ddcdb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:00,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:44:00,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:44:00,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:00,430 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52548, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:44:00,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6924ee0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:00,433 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:44:00,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:44:00,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:44:00,437 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58114, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:44:00,440 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T01:44:00,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is b93ada835ef8,36607,1732153433607 2024-11-21T01:44:00,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6d6e054b 2024-11-21T01:44:00,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T01:44:00,448 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52552, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T01:44:00,453 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514089f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:00,453 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:44:00,454 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:44:00,456 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:44:00,457 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:44:00,457 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:44:00,457 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2555cfdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:00,457 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:44:00,458 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:44:00,459 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:00,462 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52564, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:44:00,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@90ab23d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:00,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50d3f2f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:00,464 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:44:00,465 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:44:00,470 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:44:00,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:44:00,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:44:00,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d5c6971, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:00,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:44:00,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:44:00,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:00,476 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52574, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:44:00,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73a084b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:00,478 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46ad9e2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:00,478 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:44:00,479 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:44:00,480 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:44:00,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:44:00,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:44:00,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a5a580f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:00,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:44:00,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:44:00,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:00,484 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52586, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:44:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33125057, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:00,487 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6308d33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:00,487 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:44:00,487 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:44:00,490 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:44:00,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:44:00,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:44:00,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@351f00f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:00,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:44:00,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:44:00,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:00,495 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:44:00,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@383661a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:00,497 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a822d7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:00,497 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:44:00,497 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:44:00,499 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:44:00,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:44:00,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:44:00,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4a2a39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:00,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:44:00,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:44:00,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:00,501 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52634, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:44:00,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@738bed33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:00,504 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5efa0d50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:00,504 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:44:00,504 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:44:00,505 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:44:00,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:44:00,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:44:00,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50131031, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:00,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:44:00,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:44:00,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:00,510 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52654, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:44:00,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3981c165, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:00,513 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f66d26b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:00,513 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:44:00,513 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:44:00,514 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:44:00,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:44:00,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:44:00,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c0a534c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:00,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:44:00,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:44:00,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:00,517 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52676, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:44:00,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58dec3db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:00,525 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d403639, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:00,526 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:44:00,526 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:44:00,527 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:44:00,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:44:00,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:44:00,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@388f703a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:00,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:44:00,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:44:00,529 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:00,530 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52700, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:44:00,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7299e0d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:00,532 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@93b4e10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:00,532 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:44:00,532 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:44:00,533 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:44:00,534 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:44:00,534 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:44:00,534 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bb69f80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:00,534 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:44:00,534 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:44:00,535 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:00,536 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:44:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c19ce67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:00,540 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:44:00,540 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:44:00,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:44:00,542 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:44:00,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:44:00,543 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:44:00,543 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:44:00,544 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:44:00,544 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58118, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:44:00,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:44:00,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:44:00,546 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:44:00,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:44:00,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:44:00,548 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58124, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:44:00,549 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58136, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:44:00,550 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58150, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:44:00,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:44:00,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:44:00,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2] 2024-11-21T01:44:00,555 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:44:00,557 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58154, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:44:00,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2] 2024-11-21T01:44:00,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2] 2024-11-21T01:44:00,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:44:00,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2] 2024-11-21T01:44:00,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:44:00,561 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:44:00,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2] 2024-11-21T01:44:00,562 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:44:00,563 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:44:00,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:44:00,565 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:44:00,616 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:44:00,623 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:44:00,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:44:00,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:44:00,626 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:44:00,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2] 2024-11-21T01:44:00,628 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58176, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:44:00,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2] 2024-11-21T01:44:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:44:00,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2] 2024-11-21T01:44:00,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:44:00,639 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:44:00,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-21T01:44:00,647 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:44:00,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2] 2024-11-21T01:44:00,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T01:44:00,651 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:44:00,653 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:44:00,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 06db9be3c4e6682aa8966442e1ef70cc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T01:44:00,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:44:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T01:44:00,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=A 2024-11-21T01:44:00,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:00,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=B 2024-11-21T01:44:00,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:00,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=C 2024-11-21T01:44:00,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:00,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T01:44:00,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:00,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:00,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:00,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:00,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:00,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:00,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/ff226af3ec5b4b589df0a078dbef6065 is 50, key is test_row_0/A:col10/1732153440729/Put/seqid=0 2024-11-21T01:44:00,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T01:44:01,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741837_1013 (size=12001) 2024-11-21T01:44:01,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/ff226af3ec5b4b589df0a078dbef6065 2024-11-21T01:44:01,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:01,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153500954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:01,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:01,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T01:44:01,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:01,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:01,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:01,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153500976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:01,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:01,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732153440931, max timestamp=1732153440978Num uniques 859; 2024-11-21T01:44:01,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:01,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:01,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T01:44:01,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:01,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:01,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:01,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153500972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:01,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:01,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732153440931, max timestamp=1732153440978Num uniques 859; 2024-11-21T01:44:01,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T01:44:01,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732153440931, max timestamp=1732153440978Num uniques 859; 2024-11-21T01:44:01,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T01:44:01,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/aa2327f5f5244ebfa8b2844800ca58c6 is 50, key is test_row_0/B:col10/1732153440729/Put/seqid=0 2024-11-21T01:44:01,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741838_1014 (size=12001) 2024-11-21T01:44:01,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T01:44:01,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:01,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:01,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:01,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:01,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:01,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:01,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153501246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:01,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:01,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153501246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:01,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:01,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:01,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153501249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:01,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:01,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153501254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:01,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153501263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:01,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T01:44:01,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T01:44:01,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:01,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:01,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:01,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:01,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:01,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:01,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:01,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153501377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:01,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:01,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153501378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:01,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:01,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153501385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:01,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:01,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153501386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:01,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:01,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:01,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153501387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:01,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:01,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T01:44:01,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:01,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:01,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:01,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:01,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:01,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:01,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153501594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:01,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/aa2327f5f5244ebfa8b2844800ca58c6 2024-11-21T01:44:01,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:01,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:01,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153501604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:01,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:01,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:01,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:01,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153501614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:01,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153501615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:01,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:01,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:01,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153501626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:01,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:01,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/46b41407933d476cbd1c277cbaec8bf6 is 50, key is test_row_0/C:col10/1732153440729/Put/seqid=0 2024-11-21T01:44:01,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741839_1015 (size=12001) 2024-11-21T01:44:01,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/46b41407933d476cbd1c277cbaec8bf6 2024-11-21T01:44:01,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/ff226af3ec5b4b589df0a078dbef6065 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/ff226af3ec5b4b589df0a078dbef6065 2024-11-21T01:44:01,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T01:44:01,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:01,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:01,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:01,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:01,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:01,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:01,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/ff226af3ec5b4b589df0a078dbef6065, entries=150, sequenceid=13, filesize=11.7 K 2024-11-21T01:44:01,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/aa2327f5f5244ebfa8b2844800ca58c6 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/aa2327f5f5244ebfa8b2844800ca58c6 2024-11-21T01:44:01,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/aa2327f5f5244ebfa8b2844800ca58c6, entries=150, sequenceid=13, filesize=11.7 K 2024-11-21T01:44:01,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/46b41407933d476cbd1c277cbaec8bf6 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/46b41407933d476cbd1c277cbaec8bf6 2024-11-21T01:44:01,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/46b41407933d476cbd1c277cbaec8bf6, entries=150, sequenceid=13, filesize=11.7 K 2024-11-21T01:44:01,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for 06db9be3c4e6682aa8966442e1ef70cc in 1017ms, sequenceid=13, compaction requested=false 2024-11-21T01:44:01,785 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-21T01:44:01,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:01,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T01:44:01,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T01:44:01,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:01,871 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 06db9be3c4e6682aa8966442e1ef70cc 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-21T01:44:01,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=A 2024-11-21T01:44:01,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:01,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=B 2024-11-21T01:44:01,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:01,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=C 2024-11-21T01:44:01,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:01,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/5a3b6853766e43b7a8191715f0165620 is 50, key is test_row_0/A:col10/1732153441154/Put/seqid=0 2024-11-21T01:44:01,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741840_1016 (size=12001) 2024-11-21T01:44:01,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:44:01,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:01,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153501951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:01,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:01,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:01,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153501955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:01,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:01,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:01,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153501957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:01,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:01,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:01,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153501962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:01,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:01,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:01,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153501964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:02,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:02,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153502073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:02,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:02,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:02,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153502074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:02,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:02,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153502075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:02,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:02,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:02,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:02,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153502075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:02,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:02,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:02,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153502093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:02,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:02,309 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/5a3b6853766e43b7a8191715f0165620 2024-11-21T01:44:02,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:02,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153502297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:02,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:02,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153502296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:02,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:02,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:02,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153502297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:02,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:02,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:02,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153502298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:02,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:02,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:02,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:02,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153502314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:02,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:02,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/2b53b601758c4af3902acee9e4c2f95a is 50, key is test_row_0/B:col10/1732153441154/Put/seqid=0 2024-11-21T01:44:02,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741841_1017 (size=12001) 2024-11-21T01:44:02,373 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/2b53b601758c4af3902acee9e4c2f95a 2024-11-21T01:44:02,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/9beea32024024c2098d385cd39b9cf25 is 50, key is test_row_0/C:col10/1732153441154/Put/seqid=0 2024-11-21T01:44:02,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741842_1018 (size=12001) 2024-11-21T01:44:02,418 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/9beea32024024c2098d385cd39b9cf25 2024-11-21T01:44:02,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/5a3b6853766e43b7a8191715f0165620 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/5a3b6853766e43b7a8191715f0165620 2024-11-21T01:44:02,442 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/5a3b6853766e43b7a8191715f0165620, entries=150, sequenceid=48, filesize=11.7 K 2024-11-21T01:44:02,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/2b53b601758c4af3902acee9e4c2f95a as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/2b53b601758c4af3902acee9e4c2f95a 2024-11-21T01:44:02,460 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/2b53b601758c4af3902acee9e4c2f95a, entries=150, sequenceid=48, filesize=11.7 K 2024-11-21T01:44:02,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/9beea32024024c2098d385cd39b9cf25 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/9beea32024024c2098d385cd39b9cf25 2024-11-21T01:44:02,475 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/9beea32024024c2098d385cd39b9cf25, entries=150, sequenceid=48, filesize=11.7 K 2024-11-21T01:44:02,480 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 06db9be3c4e6682aa8966442e1ef70cc in 609ms, sequenceid=48, compaction requested=false 2024-11-21T01:44:02,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:02,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:02,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-21T01:44:02,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-21T01:44:02,493 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-21T01:44:02,493 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8330 sec 2024-11-21T01:44:02,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.8750 sec 2024-11-21T01:44:02,593 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-21T01:44:02,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 06db9be3c4e6682aa8966442e1ef70cc 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T01:44:02,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:44:02,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=A 2024-11-21T01:44:02,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:02,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=B 2024-11-21T01:44:02,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:02,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=C 2024-11-21T01:44:02,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:02,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/c51a88b9ac724f8d8c19eb93c56d740f is 50, key is test_row_0/A:col10/1732153441956/Put/seqid=0 2024-11-21T01:44:02,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741843_1019 (size=12001) 2024-11-21T01:44:02,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:02,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:02,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153502689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:02,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153502686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:02,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:02,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153502692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:02,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:02,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153502695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:02,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:02,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:02,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:02,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153502711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:02,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:02,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:02,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T01:44:02,816 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:44:02,817 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:44:02,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-21T01:44:02,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T01:44:02,821 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:44:02,824 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:44:02,825 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:44:02,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:02,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153502823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:02,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:02,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153502824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:02,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:02,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:02,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:02,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153502824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:02,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:02,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153502824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:02,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:02,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:02,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153502825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:02,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:02,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:02,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:02,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T01:44:02,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T01:44:02,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:02,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:02,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:02,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:02,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:02,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:03,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153503036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:03,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:03,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:03,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153503036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:03,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:03,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153503037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:03,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:03,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:03,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153503043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:03,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:03,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:03,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153503043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:03,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:03,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/c51a88b9ac724f8d8c19eb93c56d740f 2024-11-21T01:44:03,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/1543a422fe054b50895705843e33e1e0 is 50, key is test_row_0/B:col10/1732153441956/Put/seqid=0 2024-11-21T01:44:03,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741844_1020 (size=12001) 2024-11-21T01:44:03,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/1543a422fe054b50895705843e33e1e0 2024-11-21T01:44:03,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/7e4454cd9ee94e40a0d4d5e1db39013d is 50, key is test_row_0/C:col10/1732153441956/Put/seqid=0 2024-11-21T01:44:03,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T01:44:03,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T01:44:03,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:03,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:03,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:03,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:03,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:03,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:03,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741845_1021 (size=12001) 2024-11-21T01:44:03,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/7e4454cd9ee94e40a0d4d5e1db39013d 2024-11-21T01:44:03,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/c51a88b9ac724f8d8c19eb93c56d740f as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/c51a88b9ac724f8d8c19eb93c56d740f 2024-11-21T01:44:03,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/c51a88b9ac724f8d8c19eb93c56d740f, entries=150, sequenceid=61, filesize=11.7 K 2024-11-21T01:44:03,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/1543a422fe054b50895705843e33e1e0 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/1543a422fe054b50895705843e33e1e0 2024-11-21T01:44:03,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/1543a422fe054b50895705843e33e1e0, entries=150, sequenceid=61, filesize=11.7 K 2024-11-21T01:44:03,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/7e4454cd9ee94e40a0d4d5e1db39013d as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/7e4454cd9ee94e40a0d4d5e1db39013d 2024-11-21T01:44:03,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/7e4454cd9ee94e40a0d4d5e1db39013d, entries=150, sequenceid=61, filesize=11.7 K 2024-11-21T01:44:03,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 06db9be3c4e6682aa8966442e1ef70cc in 594ms, sequenceid=61, compaction requested=true 2024-11-21T01:44:03,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:03,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:44:03,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:03,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:44:03,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:03,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:44:03,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:03,241 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:03,241 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:03,288 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T01:44:03,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T01:44:03,294 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:03,294 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:03,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:03,294 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 06db9be3c4e6682aa8966442e1ef70cc 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T01:44:03,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=A 2024-11-21T01:44:03,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:03,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=B 2024-11-21T01:44:03,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:03,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=C 2024-11-21T01:44:03,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:03,303 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/B is initiating minor compaction (all files) 2024-11-21T01:44:03,304 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/B in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:03,304 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/aa2327f5f5244ebfa8b2844800ca58c6, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/2b53b601758c4af3902acee9e4c2f95a, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/1543a422fe054b50895705843e33e1e0] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=35.2 K 2024-11-21T01:44:03,305 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/A is initiating minor compaction (all files) 2024-11-21T01:44:03,305 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/A in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:03,305 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/ff226af3ec5b4b589df0a078dbef6065, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/5a3b6853766e43b7a8191715f0165620, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/c51a88b9ac724f8d8c19eb93c56d740f] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=35.2 K 2024-11-21T01:44:03,306 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting aa2327f5f5244ebfa8b2844800ca58c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732153440729 2024-11-21T01:44:03,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/d75a6c22e60c4fe28e020c304570487e is 50, key is test_row_0/A:col10/1732153442706/Put/seqid=0 2024-11-21T01:44:03,309 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b53b601758c4af3902acee9e4c2f95a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732153441154 2024-11-21T01:44:03,309 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff226af3ec5b4b589df0a078dbef6065, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732153440729 2024-11-21T01:44:03,316 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 1543a422fe054b50895705843e33e1e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732153441926 2024-11-21T01:44:03,317 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a3b6853766e43b7a8191715f0165620, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732153441154 2024-11-21T01:44:03,318 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting c51a88b9ac724f8d8c19eb93c56d740f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732153441926 2024-11-21T01:44:03,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741846_1022 (size=12001) 2024-11-21T01:44:03,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:03,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:44:03,360 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#B#compaction#11 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:03,360 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#A#compaction#10 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:03,361 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/00008e87abd04a3e912aa66a71c2bcf2 is 50, key is test_row_0/B:col10/1732153441956/Put/seqid=0 2024-11-21T01:44:03,361 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/3af4db8404554c95860f62541cc1d49a is 50, key is test_row_0/A:col10/1732153441956/Put/seqid=0 2024-11-21T01:44:03,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741847_1023 (size=12104) 2024-11-21T01:44:03,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741848_1024 (size=12104) 2024-11-21T01:44:03,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:03,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153503377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:03,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:03,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153503385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:03,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:03,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:03,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153503390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:03,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:03,412 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/00008e87abd04a3e912aa66a71c2bcf2 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/00008e87abd04a3e912aa66a71c2bcf2 2024-11-21T01:44:03,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:03,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153503395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:03,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:03,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:03,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153503399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:03,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:03,441 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/B of 06db9be3c4e6682aa8966442e1ef70cc into 00008e87abd04a3e912aa66a71c2bcf2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:03,441 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T01:44:03,444 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/B, priority=13, startTime=1732153443239; duration=0sec 2024-11-21T01:44:03,445 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:03,445 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:B 2024-11-21T01:44:03,445 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:03,448 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:03,448 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/C is initiating minor compaction (all files) 2024-11-21T01:44:03,448 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/C in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:03,449 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/46b41407933d476cbd1c277cbaec8bf6, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/9beea32024024c2098d385cd39b9cf25, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/7e4454cd9ee94e40a0d4d5e1db39013d] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=35.2 K 2024-11-21T01:44:03,450 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 46b41407933d476cbd1c277cbaec8bf6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732153440729 2024-11-21T01:44:03,451 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 9beea32024024c2098d385cd39b9cf25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732153441154 2024-11-21T01:44:03,452 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e4454cd9ee94e40a0d4d5e1db39013d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732153441926 2024-11-21T01:44:03,470 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#C#compaction#12 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:03,471 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/bf8b284189a646708c11bc09d9dae7aa is 50, key is test_row_0/C:col10/1732153441956/Put/seqid=0 2024-11-21T01:44:03,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741849_1025 (size=12104) 2024-11-21T01:44:03,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:03,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153503503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:03,506 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/bf8b284189a646708c11bc09d9dae7aa as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/bf8b284189a646708c11bc09d9dae7aa 2024-11-21T01:44:03,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:03,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:03,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153503514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:03,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153503514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:03,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:03,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:03,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153503523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:03,526 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/C of 06db9be3c4e6682aa8966442e1ef70cc into bf8b284189a646708c11bc09d9dae7aa(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:03,526 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:03,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,526 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/C, priority=13, startTime=1732153443241; duration=0sec 2024-11-21T01:44:03,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:03,526 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:03,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:03,526 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:C 2024-11-21T01:44:03,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153503524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:03,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:03,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:03,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153503714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:03,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:03,726 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/d75a6c22e60c4fe28e020c304570487e 2024-11-21T01:44:03,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:03,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153503733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:03,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:03,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:03,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153503735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:03,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:03,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153503735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:03,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:03,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:03,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:03,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153503735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:03,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:03,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:03,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/bed7e243e8c64b459084fd404eeb9d97 is 50, key is test_row_0/B:col10/1732153442706/Put/seqid=0 2024-11-21T01:44:03,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741850_1026 (size=12001) 2024-11-21T01:44:03,814 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/bed7e243e8c64b459084fd404eeb9d97 2024-11-21T01:44:03,828 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/3af4db8404554c95860f62541cc1d49a as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/3af4db8404554c95860f62541cc1d49a 2024-11-21T01:44:03,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/1884ee141c894eabbf9fb970faf3e3b8 is 50, key is test_row_0/C:col10/1732153442706/Put/seqid=0 2024-11-21T01:44:03,847 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/A of 06db9be3c4e6682aa8966442e1ef70cc into 3af4db8404554c95860f62541cc1d49a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:03,847 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:03,847 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/A, priority=13, startTime=1732153443231; duration=0sec 2024-11-21T01:44:03,848 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:03,848 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:A 2024-11-21T01:44:03,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741851_1027 (size=12001) 2024-11-21T01:44:03,883 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/1884ee141c894eabbf9fb970faf3e3b8 2024-11-21T01:44:03,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/d75a6c22e60c4fe28e020c304570487e as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/d75a6c22e60c4fe28e020c304570487e 2024-11-21T01:44:03,912 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/d75a6c22e60c4fe28e020c304570487e, entries=150, sequenceid=80, filesize=11.7 K 2024-11-21T01:44:03,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/bed7e243e8c64b459084fd404eeb9d97 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/bed7e243e8c64b459084fd404eeb9d97 2024-11-21T01:44:03,925 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/bed7e243e8c64b459084fd404eeb9d97, entries=150, sequenceid=80, filesize=11.7 K 2024-11-21T01:44:03,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/1884ee141c894eabbf9fb970faf3e3b8 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/1884ee141c894eabbf9fb970faf3e3b8 2024-11-21T01:44:03,946 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/1884ee141c894eabbf9fb970faf3e3b8, entries=150, sequenceid=80, filesize=11.7 K 2024-11-21T01:44:03,948 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 06db9be3c4e6682aa8966442e1ef70cc in 654ms, sequenceid=80, compaction requested=false 2024-11-21T01:44:03,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:03,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:03,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-21T01:44:03,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-21T01:44:03,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T01:44:03,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-21T01:44:03,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1330 sec 2024-11-21T01:44:03,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.1540 sec 2024-11-21T01:44:04,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 06db9be3c4e6682aa8966442e1ef70cc 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T01:44:04,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=A 2024-11-21T01:44:04,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:04,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=B 2024-11-21T01:44:04,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:04,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=C 2024-11-21T01:44:04,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:04,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:44:04,054 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-21T01:44:04,054 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-21T01:44:04,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/ac9eb303056a4f78836469d0a407b67f is 50, key is test_row_0/A:col10/1732153444024/Put/seqid=0 2024-11-21T01:44:04,061 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T01:44:04,061 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-21T01:44:04,062 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-21T01:44:04,062 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-21T01:44:04,069 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-21T01:44:04,069 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-21T01:44:04,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741852_1028 (size=14341) 2024-11-21T01:44:04,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/ac9eb303056a4f78836469d0a407b67f 2024-11-21T01:44:04,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:04,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/dc8fdde4342d4664b3a1c067add52dc9 is 50, key is test_row_0/B:col10/1732153444024/Put/seqid=0 2024-11-21T01:44:04,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153504103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:04,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:04,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:04,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153504105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:04,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:04,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153504100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:04,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:04,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:04,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:04,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153504117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:04,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:04,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:04,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153504117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:04,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:04,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741853_1029 (size=12001) 2024-11-21T01:44:04,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/dc8fdde4342d4664b3a1c067add52dc9 2024-11-21T01:44:04,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/1e5856b9f02b4ff0bd7e1e6b2de02f27 is 50, key is test_row_0/C:col10/1732153444024/Put/seqid=0 2024-11-21T01:44:04,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741854_1030 (size=12001) 2024-11-21T01:44:04,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/1e5856b9f02b4ff0bd7e1e6b2de02f27 2024-11-21T01:44:04,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/ac9eb303056a4f78836469d0a407b67f as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/ac9eb303056a4f78836469d0a407b67f 2024-11-21T01:44:04,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/ac9eb303056a4f78836469d0a407b67f, entries=200, sequenceid=97, filesize=14.0 K 2024-11-21T01:44:04,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/dc8fdde4342d4664b3a1c067add52dc9 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/dc8fdde4342d4664b3a1c067add52dc9 2024-11-21T01:44:04,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:04,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153504226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:04,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:04,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153504226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:04,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:04,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153504234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:04,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:04,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153504234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:04,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:04,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:04,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:04,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153504234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:04,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/dc8fdde4342d4664b3a1c067add52dc9, entries=150, sequenceid=97, filesize=11.7 K 2024-11-21T01:44:04,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/1e5856b9f02b4ff0bd7e1e6b2de02f27 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/1e5856b9f02b4ff0bd7e1e6b2de02f27 2024-11-21T01:44:04,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:04,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/1e5856b9f02b4ff0bd7e1e6b2de02f27, entries=150, sequenceid=97, filesize=11.7 K 2024-11-21T01:44:04,265 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 06db9be3c4e6682aa8966442e1ef70cc in 237ms, sequenceid=97, compaction requested=true 2024-11-21T01:44:04,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:04,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:44:04,265 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:04,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:04,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:44:04,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:04,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:44:04,265 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:04,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:04,268 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:04,268 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/A is initiating minor compaction (all files) 2024-11-21T01:44:04,268 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/A in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:04,268 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/3af4db8404554c95860f62541cc1d49a, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/d75a6c22e60c4fe28e020c304570487e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/ac9eb303056a4f78836469d0a407b67f] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=37.5 K 2024-11-21T01:44:04,269 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:04,269 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/B is initiating minor compaction (all files) 2024-11-21T01:44:04,269 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/B in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:04,269 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/00008e87abd04a3e912aa66a71c2bcf2, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/bed7e243e8c64b459084fd404eeb9d97, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/dc8fdde4342d4664b3a1c067add52dc9] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=35.3 K 2024-11-21T01:44:04,269 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3af4db8404554c95860f62541cc1d49a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732153441926 2024-11-21T01:44:04,270 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 00008e87abd04a3e912aa66a71c2bcf2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732153441926 2024-11-21T01:44:04,271 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting bed7e243e8c64b459084fd404eeb9d97, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732153442679 2024-11-21T01:44:04,272 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting dc8fdde4342d4664b3a1c067add52dc9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732153443391 2024-11-21T01:44:04,270 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting d75a6c22e60c4fe28e020c304570487e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732153442679 2024-11-21T01:44:04,275 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac9eb303056a4f78836469d0a407b67f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732153443386 2024-11-21T01:44:04,316 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#B#compaction#18 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:04,317 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/86d5e7f297f74f06a7415eefe99e012e is 50, key is test_row_0/B:col10/1732153444024/Put/seqid=0 2024-11-21T01:44:04,317 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#A#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:04,319 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/6826145630ad452897bc3501dd1b5734 is 50, key is test_row_0/A:col10/1732153444024/Put/seqid=0 2024-11-21T01:44:04,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741856_1032 (size=12207) 2024-11-21T01:44:04,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741855_1031 (size=12207) 2024-11-21T01:44:04,373 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/6826145630ad452897bc3501dd1b5734 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/6826145630ad452897bc3501dd1b5734 2024-11-21T01:44:04,386 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/A of 06db9be3c4e6682aa8966442e1ef70cc into 6826145630ad452897bc3501dd1b5734(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:04,386 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:04,386 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/A, priority=13, startTime=1732153444265; duration=0sec 2024-11-21T01:44:04,386 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:04,387 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:A 2024-11-21T01:44:04,387 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:04,389 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:04,389 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/C is initiating minor compaction (all files) 2024-11-21T01:44:04,389 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/C in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:04,390 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/bf8b284189a646708c11bc09d9dae7aa, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/1884ee141c894eabbf9fb970faf3e3b8, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/1e5856b9f02b4ff0bd7e1e6b2de02f27] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=35.3 K 2024-11-21T01:44:04,391 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf8b284189a646708c11bc09d9dae7aa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732153441926 2024-11-21T01:44:04,391 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1884ee141c894eabbf9fb970faf3e3b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732153442679 2024-11-21T01:44:04,392 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e5856b9f02b4ff0bd7e1e6b2de02f27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732153443391 2024-11-21T01:44:04,415 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#C#compaction#20 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:04,416 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/9f62f9329adb443b8bd1e10f46cd7814 is 50, key is test_row_0/C:col10/1732153444024/Put/seqid=0 2024-11-21T01:44:04,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741857_1033 (size=12207) 2024-11-21T01:44:04,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:44:04,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 06db9be3c4e6682aa8966442e1ef70cc 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T01:44:04,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=A 2024-11-21T01:44:04,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:04,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=B 2024-11-21T01:44:04,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:04,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=C 2024-11-21T01:44:04,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:04,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/1b791dec25c647bf9a1c6a92081aee9d is 50, key is test_row_0/A:col10/1732153444451/Put/seqid=0 2024-11-21T01:44:04,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:04,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153504474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:04,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:04,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153504474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:04,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:04,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:04,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153504483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:04,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:04,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:04,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153504484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:04,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:04,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:04,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153504485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:04,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:04,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741858_1034 (size=12001) 2024-11-21T01:44:04,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/1b791dec25c647bf9a1c6a92081aee9d 2024-11-21T01:44:04,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/d1ddd54ad24f411ea5ae6b3545a97104 is 50, key is test_row_0/B:col10/1732153444451/Put/seqid=0 2024-11-21T01:44:04,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741859_1035 (size=12001) 2024-11-21T01:44:04,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/d1ddd54ad24f411ea5ae6b3545a97104 2024-11-21T01:44:04,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/0cd4cc1dffdc474ca5da18b13ed046be is 50, key is test_row_0/C:col10/1732153444451/Put/seqid=0 2024-11-21T01:44:04,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741860_1036 (size=12001) 2024-11-21T01:44:04,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/0cd4cc1dffdc474ca5da18b13ed046be 2024-11-21T01:44:04,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:04,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153504594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:04,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:04,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:04,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153504594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:04,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:04,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:04,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153504594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:04,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:04,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:04,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153504595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:04,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:04,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:04,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153504595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:04,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:04,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/1b791dec25c647bf9a1c6a92081aee9d as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/1b791dec25c647bf9a1c6a92081aee9d 2024-11-21T01:44:04,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/1b791dec25c647bf9a1c6a92081aee9d, entries=150, sequenceid=118, filesize=11.7 K 2024-11-21T01:44:04,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/d1ddd54ad24f411ea5ae6b3545a97104 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/d1ddd54ad24f411ea5ae6b3545a97104 2024-11-21T01:44:04,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/d1ddd54ad24f411ea5ae6b3545a97104, entries=150, sequenceid=118, filesize=11.7 K 2024-11-21T01:44:04,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/0cd4cc1dffdc474ca5da18b13ed046be as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/0cd4cc1dffdc474ca5da18b13ed046be 2024-11-21T01:44:04,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/0cd4cc1dffdc474ca5da18b13ed046be, entries=150, sequenceid=118, filesize=11.7 K 2024-11-21T01:44:04,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 06db9be3c4e6682aa8966442e1ef70cc in 203ms, sequenceid=118, compaction requested=false 2024-11-21T01:44:04,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:04,787 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/86d5e7f297f74f06a7415eefe99e012e as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/86d5e7f297f74f06a7415eefe99e012e 2024-11-21T01:44:04,806 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/B of 06db9be3c4e6682aa8966442e1ef70cc into 86d5e7f297f74f06a7415eefe99e012e(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:04,806 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:04,806 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/B, priority=13, startTime=1732153444265; duration=0sec 2024-11-21T01:44:04,807 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:04,807 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:B 2024-11-21T01:44:04,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 06db9be3c4e6682aa8966442e1ef70cc 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T01:44:04,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:44:04,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=A 2024-11-21T01:44:04,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:04,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=B 2024-11-21T01:44:04,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:04,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=C 2024-11-21T01:44:04,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:04,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/46d4a65094fa478bbcf31f7154438652 is 50, key is test_row_0/A:col10/1732153444481/Put/seqid=0 2024-11-21T01:44:04,861 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/9f62f9329adb443b8bd1e10f46cd7814 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/9f62f9329adb443b8bd1e10f46cd7814 2024-11-21T01:44:04,878 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/C of 06db9be3c4e6682aa8966442e1ef70cc into 9f62f9329adb443b8bd1e10f46cd7814(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:04,878 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:04,878 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/C, priority=13, startTime=1732153444265; duration=0sec 2024-11-21T01:44:04,878 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:04,878 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:C 2024-11-21T01:44:04,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741861_1037 (size=12101) 2024-11-21T01:44:04,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:04,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153504890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:04,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:04,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153504914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:04,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:04,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:04,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:04,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153504919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:04,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:04,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153504938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:04,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:04,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:04,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153504940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:04,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:04,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:04,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T01:44:04,973 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:44:04,978 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:44:04,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-21T01:44:04,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T01:44:04,991 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:44:04,992 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:44:04,993 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:44:05,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:05,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153505044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:05,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:05,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153505049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:05,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:05,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153505054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:05,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:05,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:05,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:05,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:05,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:05,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153505063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:05,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:05,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:05,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:05,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:05,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153505066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:05,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:05,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:05,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:05,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T01:44:05,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T01:44:05,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:05,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:05,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:05,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:05,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:05,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:05,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:05,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153505274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:05,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:05,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:05,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:05,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/46d4a65094fa478bbcf31f7154438652 2024-11-21T01:44:05,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:05,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:05,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153505282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:05,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153505282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:05,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:05,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:05,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153505283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:05,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153505284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:05,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:05,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:05,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:05,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:05,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:05,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:05,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:05,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:05,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:05,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:05,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:05,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:05,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T01:44:05,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/d1ca337fcb4240388cce930c3a1b8497 is 50, key is test_row_0/B:col10/1732153444481/Put/seqid=0 2024-11-21T01:44:05,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T01:44:05,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:05,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:05,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:05,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:05,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:05,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:05,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741862_1038 (size=12101) 2024-11-21T01:44:05,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/d1ca337fcb4240388cce930c3a1b8497 2024-11-21T01:44:05,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/a7e775d529484c3ba8258e29d309ea77 is 50, key is test_row_0/C:col10/1732153444481/Put/seqid=0 2024-11-21T01:44:05,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741863_1039 (size=12101) 2024-11-21T01:44:05,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T01:44:05,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:05,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:05,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:05,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:05,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:05,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:05,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:05,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153505584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:05,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:05,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:05,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:05,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153505604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:05,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:05,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:05,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:05,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153505605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:05,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:05,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:05,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:05,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T01:44:05,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153505614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:05,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:05,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:05,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:05,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153505616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:05,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:05,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:05,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:05,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T01:44:05,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:05,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:05,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:05,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:05,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:05,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T01:44:05,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:05,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:05,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:05,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:05,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:05,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:05,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/a7e775d529484c3ba8258e29d309ea77 2024-11-21T01:44:05,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/46d4a65094fa478bbcf31f7154438652 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/46d4a65094fa478bbcf31f7154438652 2024-11-21T01:44:05,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/46d4a65094fa478bbcf31f7154438652, entries=150, sequenceid=133, filesize=11.8 K 2024-11-21T01:44:05,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/d1ca337fcb4240388cce930c3a1b8497 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/d1ca337fcb4240388cce930c3a1b8497 2024-11-21T01:44:05,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/d1ca337fcb4240388cce930c3a1b8497, entries=150, sequenceid=133, filesize=11.8 K 2024-11-21T01:44:05,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/a7e775d529484c3ba8258e29d309ea77 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/a7e775d529484c3ba8258e29d309ea77 2024-11-21T01:44:05,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/a7e775d529484c3ba8258e29d309ea77, entries=150, sequenceid=133, filesize=11.8 K 2024-11-21T01:44:05,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 06db9be3c4e6682aa8966442e1ef70cc in 1170ms, sequenceid=133, compaction requested=true 2024-11-21T01:44:05,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:05,995 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:05,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:44:05,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:05,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:44:05,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:05,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:44:05,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T01:44:05,997 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:05,998 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:05,998 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/A is initiating minor compaction (all files) 2024-11-21T01:44:05,998 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/A in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:05,999 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/6826145630ad452897bc3501dd1b5734, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/1b791dec25c647bf9a1c6a92081aee9d, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/46d4a65094fa478bbcf31f7154438652] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=35.5 K 2024-11-21T01:44:05,999 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6826145630ad452897bc3501dd1b5734, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732153443391 2024-11-21T01:44:06,000 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b791dec25c647bf9a1c6a92081aee9d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732153444099 2024-11-21T01:44:06,001 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46d4a65094fa478bbcf31f7154438652, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732153444476 2024-11-21T01:44:06,004 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:06,004 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/B is initiating minor compaction (all files) 2024-11-21T01:44:06,004 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/B in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:06,004 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/86d5e7f297f74f06a7415eefe99e012e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/d1ddd54ad24f411ea5ae6b3545a97104, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/d1ca337fcb4240388cce930c3a1b8497] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=35.5 K 2024-11-21T01:44:06,005 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 86d5e7f297f74f06a7415eefe99e012e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732153443391 2024-11-21T01:44:06,007 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting d1ddd54ad24f411ea5ae6b3545a97104, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732153444099 2024-11-21T01:44:06,008 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting d1ca337fcb4240388cce930c3a1b8497, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732153444476 2024-11-21T01:44:06,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T01:44:06,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:06,015 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 06db9be3c4e6682aa8966442e1ef70cc 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T01:44:06,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=A 2024-11-21T01:44:06,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:06,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=B 2024-11-21T01:44:06,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:06,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=C 2024-11-21T01:44:06,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:06,038 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#A#compaction#27 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:06,039 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/1218b22c39ae40e09c9b012016e562df is 50, key is test_row_0/A:col10/1732153444481/Put/seqid=0 2024-11-21T01:44:06,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/9113fe7a5b7847a88f4d4168fa455d39 is 50, key is test_row_0/A:col10/1732153444879/Put/seqid=0 2024-11-21T01:44:06,045 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#B#compaction#29 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:06,050 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/16163cf5c76b4fe087e4ccce74b29d24 is 50, key is test_row_0/B:col10/1732153444481/Put/seqid=0 2024-11-21T01:44:06,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741864_1040 (size=12409) 2024-11-21T01:44:06,080 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/1218b22c39ae40e09c9b012016e562df as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/1218b22c39ae40e09c9b012016e562df 2024-11-21T01:44:06,098 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/A of 06db9be3c4e6682aa8966442e1ef70cc into 1218b22c39ae40e09c9b012016e562df(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:06,098 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:06,098 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/A, priority=13, startTime=1732153445995; duration=0sec 2024-11-21T01:44:06,099 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:06,099 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:A 2024-11-21T01:44:06,099 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:06,101 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:06,101 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/C is initiating minor compaction (all files) 2024-11-21T01:44:06,101 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/C in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:06,101 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/9f62f9329adb443b8bd1e10f46cd7814, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/0cd4cc1dffdc474ca5da18b13ed046be, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/a7e775d529484c3ba8258e29d309ea77] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=35.5 K 2024-11-21T01:44:06,103 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f62f9329adb443b8bd1e10f46cd7814, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732153443391 2024-11-21T01:44:06,104 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cd4cc1dffdc474ca5da18b13ed046be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732153444099 2024-11-21T01:44:06,104 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7e775d529484c3ba8258e29d309ea77, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732153444476 2024-11-21T01:44:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:44:06,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T01:44:06,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741865_1041 (size=12151) 2024-11-21T01:44:06,133 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/9113fe7a5b7847a88f4d4168fa455d39 2024-11-21T01:44:06,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741866_1042 (size=12409) 2024-11-21T01:44:06,149 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/16163cf5c76b4fe087e4ccce74b29d24 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/16163cf5c76b4fe087e4ccce74b29d24 2024-11-21T01:44:06,161 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#C#compaction#30 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:06,161 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/B of 06db9be3c4e6682aa8966442e1ef70cc into 16163cf5c76b4fe087e4ccce74b29d24(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:06,161 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:06,162 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/B, priority=13, startTime=1732153445995; duration=0sec 2024-11-21T01:44:06,162 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:06,162 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:B 2024-11-21T01:44:06,163 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/05911bffc59c4285916009efda0b6c20 is 50, key is test_row_0/C:col10/1732153444481/Put/seqid=0 2024-11-21T01:44:06,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/c12144a988df4d2f8c650573ab2d1827 is 50, key is test_row_0/B:col10/1732153444879/Put/seqid=0 2024-11-21T01:44:06,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153506177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:06,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:06,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153506181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:06,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:06,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153506181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153506183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:06,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153506186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:06,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:06,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741868_1044 (size=12151) 2024-11-21T01:44:06,208 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/c12144a988df4d2f8c650573ab2d1827 2024-11-21T01:44:06,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741867_1043 (size=12409) 2024-11-21T01:44:06,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/51397f5a15dc457886c2183bc7260809 is 50, key is test_row_0/C:col10/1732153444879/Put/seqid=0 2024-11-21T01:44:06,243 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/05911bffc59c4285916009efda0b6c20 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/05911bffc59c4285916009efda0b6c20 2024-11-21T01:44:06,255 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/C of 06db9be3c4e6682aa8966442e1ef70cc into 05911bffc59c4285916009efda0b6c20(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:06,255 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:06,255 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/C, priority=13, startTime=1732153445997; duration=0sec 2024-11-21T01:44:06,255 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:06,255 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:C 2024-11-21T01:44:06,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741869_1045 (size=12151) 2024-11-21T01:44:06,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153506294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:06,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:06,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153506304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:06,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153506306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:06,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153506306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:06,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:06,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:06,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:06,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153506318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:06,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:06,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153506513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:06,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:06,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:06,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153506523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:06,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:06,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153506524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:06,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:06,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153506525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:06,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:06,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:06,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:06,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:06,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153506533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:06,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:06,683 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/51397f5a15dc457886c2183bc7260809 2024-11-21T01:44:06,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/9113fe7a5b7847a88f4d4168fa455d39 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/9113fe7a5b7847a88f4d4168fa455d39 2024-11-21T01:44:06,706 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/9113fe7a5b7847a88f4d4168fa455d39, entries=150, sequenceid=153, filesize=11.9 K 2024-11-21T01:44:06,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/c12144a988df4d2f8c650573ab2d1827 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/c12144a988df4d2f8c650573ab2d1827 2024-11-21T01:44:06,722 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/c12144a988df4d2f8c650573ab2d1827, entries=150, sequenceid=153, filesize=11.9 K 2024-11-21T01:44:06,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/51397f5a15dc457886c2183bc7260809 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/51397f5a15dc457886c2183bc7260809 2024-11-21T01:44:06,735 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/51397f5a15dc457886c2183bc7260809, entries=150, sequenceid=153, filesize=11.9 K 2024-11-21T01:44:06,743 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 06db9be3c4e6682aa8966442e1ef70cc in 729ms, sequenceid=153, compaction requested=false 2024-11-21T01:44:06,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:06,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:06,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-21T01:44:06,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-21T01:44:06,750 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-21T01:44:06,750 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7530 sec 2024-11-21T01:44:06,753 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.7720 sec 2024-11-21T01:44:06,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 06db9be3c4e6682aa8966442e1ef70cc 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-21T01:44:06,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=A 2024-11-21T01:44:06,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:06,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=B 2024-11-21T01:44:06,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:06,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=C 2024-11-21T01:44:06,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:06,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:44:06,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/a2201d51813448f6aa08feeda06a5564 is 50, key is test_row_0/A:col10/1732153446825/Put/seqid=0 2024-11-21T01:44:06,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741870_1046 (size=12151) 2024-11-21T01:44:06,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/a2201d51813448f6aa08feeda06a5564 2024-11-21T01:44:06,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:06,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153506860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:06,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:06,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:06,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153506864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:06,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:06,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153506866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:06,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:06,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:06,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:06,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153506869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:06,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:06,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153506872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:06,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/d0fd7af99971427c87ac2a2e61ee5929 is 50, key is test_row_0/B:col10/1732153446825/Put/seqid=0 2024-11-21T01:44:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:06,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:06,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741871_1047 (size=12151) 2024-11-21T01:44:06,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/d0fd7af99971427c87ac2a2e61ee5929 2024-11-21T01:44:06,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/856c3386b59e4a5b81d5739d34631049 is 50, key is test_row_0/C:col10/1732153446825/Put/seqid=0 2024-11-21T01:44:06,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741872_1048 (size=12151) 2024-11-21T01:44:06,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/856c3386b59e4a5b81d5739d34631049 2024-11-21T01:44:06,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/a2201d51813448f6aa08feeda06a5564 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/a2201d51813448f6aa08feeda06a5564 2024-11-21T01:44:06,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:06,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153506984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:06,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:06,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153506984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:06,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:06,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153506984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:06,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:06,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153506993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:06,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:06,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153506996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:06,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:06,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:07,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/a2201d51813448f6aa08feeda06a5564, entries=150, sequenceid=171, filesize=11.9 K 2024-11-21T01:44:07,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/d0fd7af99971427c87ac2a2e61ee5929 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/d0fd7af99971427c87ac2a2e61ee5929 2024-11-21T01:44:07,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/d0fd7af99971427c87ac2a2e61ee5929, entries=150, sequenceid=171, filesize=11.9 K 2024-11-21T01:44:07,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/856c3386b59e4a5b81d5739d34631049 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/856c3386b59e4a5b81d5739d34631049 2024-11-21T01:44:07,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/856c3386b59e4a5b81d5739d34631049, entries=150, sequenceid=171, filesize=11.9 K 2024-11-21T01:44:07,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 06db9be3c4e6682aa8966442e1ef70cc in 224ms, sequenceid=171, compaction requested=true 2024-11-21T01:44:07,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:07,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:44:07,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:07,052 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:07,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:44:07,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:07,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:44:07,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:07,052 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:07,055 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:07,056 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/B is initiating minor compaction (all files) 2024-11-21T01:44:07,055 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:07,056 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/B in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:07,056 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/A is initiating minor compaction (all files) 2024-11-21T01:44:07,056 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/A in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:07,056 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/16163cf5c76b4fe087e4ccce74b29d24, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/c12144a988df4d2f8c650573ab2d1827, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/d0fd7af99971427c87ac2a2e61ee5929] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=35.9 K 2024-11-21T01:44:07,056 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/1218b22c39ae40e09c9b012016e562df, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/9113fe7a5b7847a88f4d4168fa455d39, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/a2201d51813448f6aa08feeda06a5564] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=35.9 K 2024-11-21T01:44:07,059 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1218b22c39ae40e09c9b012016e562df, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732153444476 2024-11-21T01:44:07,060 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9113fe7a5b7847a88f4d4168fa455d39, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732153444879 2024-11-21T01:44:07,060 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 16163cf5c76b4fe087e4ccce74b29d24, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732153444476 2024-11-21T01:44:07,061 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2201d51813448f6aa08feeda06a5564, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732153446171 2024-11-21T01:44:07,062 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting c12144a988df4d2f8c650573ab2d1827, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732153444879 2024-11-21T01:44:07,064 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting d0fd7af99971427c87ac2a2e61ee5929, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732153446171 2024-11-21T01:44:07,092 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#B#compaction#37 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:07,093 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/f5cdfa4a1c7f47e8bc362b1e5ed53d18 is 50, key is test_row_0/B:col10/1732153446825/Put/seqid=0 2024-11-21T01:44:07,096 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#A#compaction#36 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:07,098 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/58bdc8b25b494fc1acbfb5eede8f7455 is 50, key is test_row_0/A:col10/1732153446825/Put/seqid=0 2024-11-21T01:44:07,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741874_1050 (size=12561) 2024-11-21T01:44:07,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741873_1049 (size=12561) 2024-11-21T01:44:07,130 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/58bdc8b25b494fc1acbfb5eede8f7455 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/58bdc8b25b494fc1acbfb5eede8f7455 2024-11-21T01:44:07,135 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/f5cdfa4a1c7f47e8bc362b1e5ed53d18 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/f5cdfa4a1c7f47e8bc362b1e5ed53d18 2024-11-21T01:44:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T01:44:07,144 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/A of 06db9be3c4e6682aa8966442e1ef70cc into 58bdc8b25b494fc1acbfb5eede8f7455(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:07,144 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:07,144 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/A, priority=13, startTime=1732153447052; duration=0sec 2024-11-21T01:44:07,144 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:07,144 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:A 2024-11-21T01:44:07,144 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:07,145 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:44:07,146 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:44:07,148 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:07,148 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/C is initiating minor compaction (all files) 2024-11-21T01:44:07,148 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/C in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:07,149 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/05911bffc59c4285916009efda0b6c20, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/51397f5a15dc457886c2183bc7260809, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/856c3386b59e4a5b81d5739d34631049] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=35.9 K 2024-11-21T01:44:07,150 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05911bffc59c4285916009efda0b6c20, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732153444476 2024-11-21T01:44:07,150 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51397f5a15dc457886c2183bc7260809, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732153444879 2024-11-21T01:44:07,151 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 856c3386b59e4a5b81d5739d34631049, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732153446171 2024-11-21T01:44:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-21T01:44:07,175 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/B of 06db9be3c4e6682aa8966442e1ef70cc into f5cdfa4a1c7f47e8bc362b1e5ed53d18(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:07,175 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:07,175 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/B, priority=13, startTime=1732153447052; duration=0sec 2024-11-21T01:44:07,176 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:07,176 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:B 2024-11-21T01:44:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T01:44:07,177 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:44:07,179 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:44:07,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:44:07,181 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#C#compaction#38 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:07,182 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/221c9f218b0b4951aaa041fa2ded3e7e is 50, key is test_row_0/C:col10/1732153446825/Put/seqid=0 2024-11-21T01:44:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:44:07,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 06db9be3c4e6682aa8966442e1ef70cc 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-21T01:44:07,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=A 2024-11-21T01:44:07,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:07,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=B 2024-11-21T01:44:07,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:07,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=C 2024-11-21T01:44:07,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:07,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/ddb1c0d58b794e8e848eddf49afdf8e9 is 50, key is test_row_0/A:col10/1732153447194/Put/seqid=0 2024-11-21T01:44:07,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153507226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:07,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153507227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:07,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:07,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153507228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:07,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:07,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:07,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153507229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:07,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:07,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153507231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:07,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:07,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:07,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741876_1052 (size=12151) 2024-11-21T01:44:07,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/ddb1c0d58b794e8e848eddf49afdf8e9 2024-11-21T01:44:07,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741875_1051 (size=12561) 2024-11-21T01:44:07,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/13bf9abbb76742da86e230efa77be623 is 50, key is test_row_0/B:col10/1732153447194/Put/seqid=0 2024-11-21T01:44:07,280 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/221c9f218b0b4951aaa041fa2ded3e7e as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/221c9f218b0b4951aaa041fa2ded3e7e 2024-11-21T01:44:07,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T01:44:07,294 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/C of 06db9be3c4e6682aa8966442e1ef70cc into 221c9f218b0b4951aaa041fa2ded3e7e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:07,294 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:07,294 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/C, priority=13, startTime=1732153447052; duration=0sec 2024-11-21T01:44:07,294 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:07,294 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:C 2024-11-21T01:44:07,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741877_1053 (size=12151) 2024-11-21T01:44:07,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T01:44:07,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:07,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:07,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:07,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:07,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:07,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:07,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153507349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:07,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:07,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153507349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:07,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:07,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:07,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153507352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:07,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:07,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:07,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:07,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153507366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:07,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:07,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153507366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:07,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:07,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T01:44:07,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T01:44:07,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:07,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:07,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:07,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:07,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:07,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:07,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:07,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153507563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:07,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:07,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:07,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153507564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:07,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:07,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153507563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:07,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:07,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:07,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:07,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153507574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:07,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:07,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153507574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:07,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:07,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:07,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T01:44:07,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:07,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:07,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:07,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:07,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:07,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:07,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/13bf9abbb76742da86e230efa77be623 2024-11-21T01:44:07,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/70e78e3c4ce346aaa0f6f06f0b9f7fc0 is 50, key is test_row_0/C:col10/1732153447194/Put/seqid=0 2024-11-21T01:44:07,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741878_1054 (size=12151) 2024-11-21T01:44:07,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/70e78e3c4ce346aaa0f6f06f0b9f7fc0 2024-11-21T01:44:07,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/ddb1c0d58b794e8e848eddf49afdf8e9 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/ddb1c0d58b794e8e848eddf49afdf8e9 2024-11-21T01:44:07,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/ddb1c0d58b794e8e848eddf49afdf8e9, entries=150, sequenceid=191, filesize=11.9 K 2024-11-21T01:44:07,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/13bf9abbb76742da86e230efa77be623 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/13bf9abbb76742da86e230efa77be623 2024-11-21T01:44:07,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/13bf9abbb76742da86e230efa77be623, entries=150, sequenceid=191, filesize=11.9 K 2024-11-21T01:44:07,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/70e78e3c4ce346aaa0f6f06f0b9f7fc0 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/70e78e3c4ce346aaa0f6f06f0b9f7fc0 2024-11-21T01:44:07,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T01:44:07,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/70e78e3c4ce346aaa0f6f06f0b9f7fc0, entries=150, sequenceid=191, filesize=11.9 K 2024-11-21T01:44:07,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 06db9be3c4e6682aa8966442e1ef70cc in 613ms, sequenceid=191, compaction requested=false 2024-11-21T01:44:07,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:07,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T01:44:07,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:07,838 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 06db9be3c4e6682aa8966442e1ef70cc 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T01:44:07,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=A 2024-11-21T01:44:07,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:07,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=B 2024-11-21T01:44:07,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:07,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=C 2024-11-21T01:44:07,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:07,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/401069ea60404d94a5f9b749f2d11e04 is 50, key is test_row_0/A:col10/1732153447226/Put/seqid=0 2024-11-21T01:44:07,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741879_1055 (size=12151) 2024-11-21T01:44:07,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:44:07,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:07,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:07,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:07,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153507905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:07,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153507907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:07,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:07,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:07,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153507911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:07,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:07,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153507909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:07,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:07,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:07,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153507915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:07,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:07,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:07,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:07,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:08,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:08,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153508024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:08,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:08,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:08,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153508024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:08,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153508024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:08,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:08,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:08,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153508024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:08,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:08,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153508024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:08,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:08,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:08,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:08,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153508233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:08,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153508234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:08,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:08,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153508234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:08,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:08,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153508234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:08,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:08,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153508234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:08,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:08,261 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/401069ea60404d94a5f9b749f2d11e04 2024-11-21T01:44:08,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/8ca5e280b27542ceb7504b882835d0a5 is 50, key is test_row_0/B:col10/1732153447226/Put/seqid=0 2024-11-21T01:44:08,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741880_1056 (size=12151) 2024-11-21T01:44:08,310 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/8ca5e280b27542ceb7504b882835d0a5 2024-11-21T01:44:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T01:44:08,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/c7822297793e4df19442051e4beae45f is 50, key is test_row_0/C:col10/1732153447226/Put/seqid=0 2024-11-21T01:44:08,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741881_1057 (size=12151) 2024-11-21T01:44:08,349 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/c7822297793e4df19442051e4beae45f 2024-11-21T01:44:08,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/401069ea60404d94a5f9b749f2d11e04 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/401069ea60404d94a5f9b749f2d11e04 2024-11-21T01:44:08,378 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/401069ea60404d94a5f9b749f2d11e04, entries=150, sequenceid=206, filesize=11.9 K 2024-11-21T01:44:08,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/8ca5e280b27542ceb7504b882835d0a5 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/8ca5e280b27542ceb7504b882835d0a5 2024-11-21T01:44:08,394 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/8ca5e280b27542ceb7504b882835d0a5, entries=150, sequenceid=206, filesize=11.9 K 2024-11-21T01:44:08,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/c7822297793e4df19442051e4beae45f as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/c7822297793e4df19442051e4beae45f 2024-11-21T01:44:08,417 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/c7822297793e4df19442051e4beae45f, entries=150, sequenceid=206, filesize=11.9 K 2024-11-21T01:44:08,418 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 06db9be3c4e6682aa8966442e1ef70cc in 580ms, sequenceid=206, compaction requested=true 2024-11-21T01:44:08,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:08,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:08,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-21T01:44:08,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-21T01:44:08,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-21T01:44:08,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2420 sec 2024-11-21T01:44:08,429 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.2810 sec 2024-11-21T01:44:08,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732153447904, max timestamp=1732153448562Num uniques 859; 2024-11-21T01:44:08,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T01:44:08,574 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 06db9be3c4e6682aa8966442e1ef70cc 3/3 column families, dataSize=93.93 KB heapSize=272.27 KB 2024-11-21T01:44:08,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=A 2024-11-21T01:44:08,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:08,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=B 2024-11-21T01:44:08,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:08,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=C 2024-11-21T01:44:08,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:08,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:44:08,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/06f3c9502dbb47eabe4d0f8d4600d747 is 50, key is test_row_0/A:col10/1732153447911/Put/seqid=0 2024-11-21T01:44:08,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:08,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153508608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:08,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:08,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:08,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153508610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:08,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:08,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:08,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153508613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:08,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:08,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153508613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:08,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:08,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:08,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:08,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153508622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:08,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741882_1058 (size=12151) 2024-11-21T01:44:08,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/06f3c9502dbb47eabe4d0f8d4600d747 2024-11-21T01:44:08,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/37a47146e6834d5daf3933276c587573 is 50, key is test_row_0/B:col10/1732153447911/Put/seqid=0 2024-11-21T01:44:08,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741883_1059 (size=12151) 2024-11-21T01:44:08,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/37a47146e6834d5daf3933276c587573 2024-11-21T01:44:08,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:08,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153508733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:08,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:08,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:08,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153508743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:08,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:08,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:08,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153508744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:08,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:08,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:08,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153508744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:08,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:08,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:08,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153508744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:08,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/cb03140dc82c4e5cbf1f2178fc7ad468 is 50, key is test_row_0/C:col10/1732153447911/Put/seqid=0 2024-11-21T01:44:08,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741884_1060 (size=12151) 2024-11-21T01:44:08,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/cb03140dc82c4e5cbf1f2178fc7ad468 2024-11-21T01:44:08,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/06f3c9502dbb47eabe4d0f8d4600d747 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/06f3c9502dbb47eabe4d0f8d4600d747 2024-11-21T01:44:08,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/06f3c9502dbb47eabe4d0f8d4600d747, entries=150, sequenceid=227, filesize=11.9 K 2024-11-21T01:44:08,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/37a47146e6834d5daf3933276c587573 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/37a47146e6834d5daf3933276c587573 2024-11-21T01:44:08,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/37a47146e6834d5daf3933276c587573, entries=150, sequenceid=227, filesize=11.9 K 2024-11-21T01:44:08,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/cb03140dc82c4e5cbf1f2178fc7ad468 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/cb03140dc82c4e5cbf1f2178fc7ad468 2024-11-21T01:44:08,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/cb03140dc82c4e5cbf1f2178fc7ad468, entries=150, sequenceid=227, filesize=11.9 K 2024-11-21T01:44:08,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~272.22 KB/278752, currentSize=80.51 KB/82440 for 06db9be3c4e6682aa8966442e1ef70cc in 292ms, sequenceid=227, compaction requested=true 2024-11-21T01:44:08,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:08,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:44:08,867 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:44:08,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:08,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:44:08,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:08,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:44:08,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T01:44:08,867 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:44:08,871 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:44:08,871 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:44:08,872 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/B is initiating minor compaction (all files) 2024-11-21T01:44:08,872 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/A is initiating minor compaction (all files) 2024-11-21T01:44:08,872 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/B in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:08,872 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/A in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:08,872 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/f5cdfa4a1c7f47e8bc362b1e5ed53d18, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/13bf9abbb76742da86e230efa77be623, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/8ca5e280b27542ceb7504b882835d0a5, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/37a47146e6834d5daf3933276c587573] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=47.9 K 2024-11-21T01:44:08,872 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/58bdc8b25b494fc1acbfb5eede8f7455, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/ddb1c0d58b794e8e848eddf49afdf8e9, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/401069ea60404d94a5f9b749f2d11e04, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/06f3c9502dbb47eabe4d0f8d4600d747] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=47.9 K 2024-11-21T01:44:08,873 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting f5cdfa4a1c7f47e8bc362b1e5ed53d18, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732153446171 2024-11-21T01:44:08,873 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58bdc8b25b494fc1acbfb5eede8f7455, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732153446171 2024-11-21T01:44:08,874 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 13bf9abbb76742da86e230efa77be623, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732153446849 2024-11-21T01:44:08,874 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddb1c0d58b794e8e848eddf49afdf8e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732153446849 2024-11-21T01:44:08,874 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ca5e280b27542ceb7504b882835d0a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732153447224 2024-11-21T01:44:08,875 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 37a47146e6834d5daf3933276c587573, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732153447911 2024-11-21T01:44:08,875 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 401069ea60404d94a5f9b749f2d11e04, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732153447224 2024-11-21T01:44:08,876 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06f3c9502dbb47eabe4d0f8d4600d747, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732153447911 2024-11-21T01:44:08,912 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#A#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:08,913 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/3a003c18ebc64ddfb1b4480dd42259fd is 50, key is test_row_0/A:col10/1732153447911/Put/seqid=0 2024-11-21T01:44:08,916 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#B#compaction#48 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:08,917 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/df9e161c694548c1bb558ab2407ecae0 is 50, key is test_row_0/B:col10/1732153447911/Put/seqid=0 2024-11-21T01:44:08,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741886_1062 (size=12697) 2024-11-21T01:44:08,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741885_1061 (size=12697) 2024-11-21T01:44:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:44:08,965 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 06db9be3c4e6682aa8966442e1ef70cc 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-21T01:44:08,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=A 2024-11-21T01:44:08,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:08,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=B 2024-11-21T01:44:08,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:08,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=C 2024-11-21T01:44:08,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:08,987 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/df9e161c694548c1bb558ab2407ecae0 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/df9e161c694548c1bb558ab2407ecae0 2024-11-21T01:44:08,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/1acb2875648244b6b469c12e837a962a is 50, key is test_row_0/A:col10/1732153448961/Put/seqid=0 2024-11-21T01:44:08,997 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/3a003c18ebc64ddfb1b4480dd42259fd as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/3a003c18ebc64ddfb1b4480dd42259fd 2024-11-21T01:44:09,010 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/B of 06db9be3c4e6682aa8966442e1ef70cc into df9e161c694548c1bb558ab2407ecae0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:09,010 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:09,011 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/B, priority=12, startTime=1732153448867; duration=0sec 2024-11-21T01:44:09,011 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:09,011 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:B 2024-11-21T01:44:09,011 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:44:09,012 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/A of 06db9be3c4e6682aa8966442e1ef70cc into 3a003c18ebc64ddfb1b4480dd42259fd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:09,012 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:09,012 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/A, priority=12, startTime=1732153448867; duration=0sec 2024-11-21T01:44:09,012 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:09,012 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:A 2024-11-21T01:44:09,013 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:44:09,013 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/C is initiating minor compaction (all files) 2024-11-21T01:44:09,013 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/C in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:09,014 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/221c9f218b0b4951aaa041fa2ded3e7e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/70e78e3c4ce346aaa0f6f06f0b9f7fc0, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/c7822297793e4df19442051e4beae45f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/cb03140dc82c4e5cbf1f2178fc7ad468] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=47.9 K 2024-11-21T01:44:09,014 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 221c9f218b0b4951aaa041fa2ded3e7e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732153446171 2024-11-21T01:44:09,015 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 70e78e3c4ce346aaa0f6f06f0b9f7fc0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732153446849 2024-11-21T01:44:09,016 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting c7822297793e4df19442051e4beae45f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732153447224 2024-11-21T01:44:09,016 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting cb03140dc82c4e5cbf1f2178fc7ad468, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732153447911 2024-11-21T01:44:09,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153509011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:09,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:09,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:09,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153509025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153509026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:09,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:09,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153509032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:09,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:09,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:09,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:09,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153509034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:09,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:09,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:09,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741887_1063 (size=12147) 2024-11-21T01:44:09,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/1acb2875648244b6b469c12e837a962a 2024-11-21T01:44:09,058 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#C#compaction#51 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:09,059 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/98260bc22c7c4257b6e93bbc0e85a2df is 50, key is test_row_0/C:col10/1732153447911/Put/seqid=0 2024-11-21T01:44:09,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/bd3258646b3f4ba59b41f6fc07fb21ec is 50, key is test_row_0/B:col10/1732153448961/Put/seqid=0 2024-11-21T01:44:09,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741888_1064 (size=9757) 2024-11-21T01:44:09,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741889_1065 (size=12697) 2024-11-21T01:44:09,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153509143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:09,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:09,151 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/98260bc22c7c4257b6e93bbc0e85a2df as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/98260bc22c7c4257b6e93bbc0e85a2df 2024-11-21T01:44:09,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:09,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153509153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:09,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:09,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:09,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153509153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:09,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:09,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153509153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:09,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:09,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:09,165 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/C of 06db9be3c4e6682aa8966442e1ef70cc into 98260bc22c7c4257b6e93bbc0e85a2df(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:09,165 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:09,166 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/C, priority=12, startTime=1732153448867; duration=0sec 2024-11-21T01:44:09,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153509153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:09,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:09,170 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:09,170 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:C 2024-11-21T01:44:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T01:44:09,336 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:44:09,336 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:44:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-21T01:44:09,340 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:44:09,342 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:44:09,342 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:44:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T01:44:09,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153509356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:09,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:09,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153509366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:09,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:09,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153509374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:09,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:09,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153509375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:09,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:09,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153509375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:09,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:09,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:09,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T01:44:09,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T01:44:09,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:09,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:09,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:09,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:09,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:09,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:09,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/bd3258646b3f4ba59b41f6fc07fb21ec 2024-11-21T01:44:09,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/66e5280aad65408dbd09414e6bcd093f is 50, key is test_row_0/C:col10/1732153448961/Put/seqid=0 2024-11-21T01:44:09,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741890_1066 (size=9757) 2024-11-21T01:44:09,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/66e5280aad65408dbd09414e6bcd093f 2024-11-21T01:44:09,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/1acb2875648244b6b469c12e837a962a as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/1acb2875648244b6b469c12e837a962a 2024-11-21T01:44:09,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/1acb2875648244b6b469c12e837a962a, entries=150, sequenceid=243, filesize=11.9 K 2024-11-21T01:44:09,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/bd3258646b3f4ba59b41f6fc07fb21ec as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/bd3258646b3f4ba59b41f6fc07fb21ec 2024-11-21T01:44:09,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/bd3258646b3f4ba59b41f6fc07fb21ec, entries=100, sequenceid=243, filesize=9.5 K 2024-11-21T01:44:09,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/66e5280aad65408dbd09414e6bcd093f as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/66e5280aad65408dbd09414e6bcd093f 2024-11-21T01:44:09,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T01:44:09,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:09,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:09,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:09,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:09,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:09,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/66e5280aad65408dbd09414e6bcd093f, entries=100, sequenceid=243, filesize=9.5 K 2024-11-21T01:44:09,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:09,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T01:44:09,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 06db9be3c4e6682aa8966442e1ef70cc in 698ms, sequenceid=243, compaction requested=false 2024-11-21T01:44:09,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:09,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 06db9be3c4e6682aa8966442e1ef70cc 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-21T01:44:09,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=A 2024-11-21T01:44:09,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:09,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=B 2024-11-21T01:44:09,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:09,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=C 2024-11-21T01:44:09,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:09,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:44:09,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/16c408688faf483993eb388de90d2c41 is 50, key is test_row_0/A:col10/1732153449014/Put/seqid=0 2024-11-21T01:44:09,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741891_1067 (size=14741) 2024-11-21T01:44:09,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:09,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153509744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:09,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:09,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:09,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153509753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:09,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:09,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:09,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153509755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:09,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:09,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:09,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153509755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:09,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153509758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:09,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:09,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T01:44:09,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:09,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:09,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:09,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:09,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:09,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:09,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:09,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153509863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:09,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:09,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:09,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153509864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:09,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:09,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:09,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153509864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:09,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:09,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153509886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:09,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:09,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153509886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:09,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:09,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:09,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:09,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T01:44:09,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T01:44:09,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:09,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:09,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:09,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:09,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:09,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:10,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:10,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153510073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:10,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:10,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153510074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:10,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:10,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:10,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:10,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153510083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:10,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:10,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:10,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153510103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:10,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:10,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:10,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153510113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:10,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:10,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T01:44:10,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:10,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:10,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:10,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:10,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:10,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:10,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/16c408688faf483993eb388de90d2c41 2024-11-21T01:44:10,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/8d01f5c435a0467eaca071b120d2fd0b is 50, key is test_row_0/B:col10/1732153449014/Put/seqid=0 2024-11-21T01:44:10,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741892_1068 (size=12301) 2024-11-21T01:44:10,253 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/8d01f5c435a0467eaca071b120d2fd0b 2024-11-21T01:44:10,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/7179161c07744e90b2904e16699401d0 is 50, key is test_row_0/C:col10/1732153449014/Put/seqid=0 2024-11-21T01:44:10,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T01:44:10,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:10,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:10,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:10,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:10,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:10,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:10,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741893_1069 (size=12301) 2024-11-21T01:44:10,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/7179161c07744e90b2904e16699401d0 2024-11-21T01:44:10,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/16c408688faf483993eb388de90d2c41 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/16c408688faf483993eb388de90d2c41 2024-11-21T01:44:10,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/16c408688faf483993eb388de90d2c41, entries=200, sequenceid=263, filesize=14.4 K 2024-11-21T01:44:10,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/8d01f5c435a0467eaca071b120d2fd0b as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/8d01f5c435a0467eaca071b120d2fd0b 2024-11-21T01:44:10,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/8d01f5c435a0467eaca071b120d2fd0b, entries=150, sequenceid=263, filesize=12.0 K 2024-11-21T01:44:10,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/7179161c07744e90b2904e16699401d0 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/7179161c07744e90b2904e16699401d0 2024-11-21T01:44:10,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/7179161c07744e90b2904e16699401d0, entries=150, sequenceid=263, filesize=12.0 K 2024-11-21T01:44:10,393 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 06db9be3c4e6682aa8966442e1ef70cc in 708ms, sequenceid=263, compaction requested=true 2024-11-21T01:44:10,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:10,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:44:10,394 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:10,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:10,394 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:10,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:44:10,396 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:10,396 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/A is initiating minor compaction (all files) 2024-11-21T01:44:10,396 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/A in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:10,396 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/3a003c18ebc64ddfb1b4480dd42259fd, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/1acb2875648244b6b469c12e837a962a, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/16c408688faf483993eb388de90d2c41] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=38.7 K 2024-11-21T01:44:10,397 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:10,397 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/B is initiating minor compaction (all files) 2024-11-21T01:44:10,397 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a003c18ebc64ddfb1b4480dd42259fd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732153447911 2024-11-21T01:44:10,397 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/B in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:10,397 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/df9e161c694548c1bb558ab2407ecae0, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/bd3258646b3f4ba59b41f6fc07fb21ec, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/8d01f5c435a0467eaca071b120d2fd0b] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=33.9 K 2024-11-21T01:44:10,398 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting df9e161c694548c1bb558ab2407ecae0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732153447911 2024-11-21T01:44:10,398 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1acb2875648244b6b469c12e837a962a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732153448609 2024-11-21T01:44:10,398 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16c408688faf483993eb388de90d2c41, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732153449014 2024-11-21T01:44:10,398 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting bd3258646b3f4ba59b41f6fc07fb21ec, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732153448619 2024-11-21T01:44:10,399 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d01f5c435a0467eaca071b120d2fd0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732153449014 2024-11-21T01:44:10,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:44:10,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:10,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:44:10,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:10,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 06db9be3c4e6682aa8966442e1ef70cc 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-21T01:44:10,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=A 2024-11-21T01:44:10,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:10,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=B 2024-11-21T01:44:10,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:10,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=C 2024-11-21T01:44:10,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:10,417 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#B#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:10,418 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/54b918c104bf48a3948e22135422f52c is 50, key is test_row_0/B:col10/1732153449014/Put/seqid=0 2024-11-21T01:44:10,422 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#A#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:10,422 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/16a83b65d1ec4491a9f768418a6d74f1 is 50, key is test_row_0/A:col10/1732153449014/Put/seqid=0 2024-11-21T01:44:10,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/3a94f45df01d4375b3bcf3b3f8b41a37 is 50, key is test_row_0/A:col10/1732153449747/Put/seqid=0 2024-11-21T01:44:10,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741894_1070 (size=12949) 2024-11-21T01:44:10,447 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/54b918c104bf48a3948e22135422f52c as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/54b918c104bf48a3948e22135422f52c 2024-11-21T01:44:10,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:10,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153510434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:10,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:10,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:10,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153510439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:10,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:10,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:10,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153510446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:10,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:10,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T01:44:10,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:10,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:10,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:10,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:10,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:10,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:10,456 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/B of 06db9be3c4e6682aa8966442e1ef70cc into 54b918c104bf48a3948e22135422f52c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:10,456 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:10,456 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/B, priority=13, startTime=1732153450394; duration=0sec 2024-11-21T01:44:10,456 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:10,456 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:B 2024-11-21T01:44:10,456 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:10,458 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:10,458 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/C is initiating minor compaction (all files) 2024-11-21T01:44:10,458 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/C in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:10,458 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/98260bc22c7c4257b6e93bbc0e85a2df, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/66e5280aad65408dbd09414e6bcd093f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/7179161c07744e90b2904e16699401d0] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=33.9 K 2024-11-21T01:44:10,459 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 98260bc22c7c4257b6e93bbc0e85a2df, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732153447911 2024-11-21T01:44:10,460 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 66e5280aad65408dbd09414e6bcd093f, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732153448619 2024-11-21T01:44:10,461 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 7179161c07744e90b2904e16699401d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732153449014 2024-11-21T01:44:10,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:10,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153510450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:10,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:10,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153510450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:10,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741895_1071 (size=14741) 2024-11-21T01:44:10,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:10,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T01:44:10,485 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#C#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:10,486 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/001bd063bf1040468625035b1a337510 is 50, key is test_row_0/C:col10/1732153449014/Put/seqid=0 2024-11-21T01:44:10,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741896_1072 (size=12949) 2024-11-21T01:44:10,512 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/16a83b65d1ec4491a9f768418a6d74f1 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/16a83b65d1ec4491a9f768418a6d74f1 2024-11-21T01:44:10,523 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/A of 06db9be3c4e6682aa8966442e1ef70cc into 16a83b65d1ec4491a9f768418a6d74f1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:10,523 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:10,523 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/A, priority=13, startTime=1732153450394; duration=0sec 2024-11-21T01:44:10,524 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:10,524 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:A 2024-11-21T01:44:10,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741897_1073 (size=12949) 2024-11-21T01:44:10,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:10,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153510559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:10,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:10,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:10,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153510564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:10,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:10,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:10,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153510565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:10,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:10,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153510583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:10,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153510584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:10,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T01:44:10,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:10,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:10,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:10,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:10,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:10,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:10,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T01:44:10,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:10,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:10,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:10,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:10,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:10,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:10,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:10,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153510774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:10,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:10,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:10,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153510774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:10,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:10,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153510774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:10,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:10,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:10,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:10,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153510794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:10,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:10,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153510798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:10,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:10,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:10,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:10,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/3a94f45df01d4375b3bcf3b3f8b41a37 2024-11-21T01:44:10,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/690a1385833348d393fbc9138b4d4d74 is 50, key is test_row_0/B:col10/1732153449747/Put/seqid=0 2024-11-21T01:44:10,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T01:44:10,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:10,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:10,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:10,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:10,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:10,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:10,943 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/001bd063bf1040468625035b1a337510 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/001bd063bf1040468625035b1a337510 2024-11-21T01:44:10,951 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/C of 06db9be3c4e6682aa8966442e1ef70cc into 001bd063bf1040468625035b1a337510(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:10,951 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:10,951 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/C, priority=13, startTime=1732153450400; duration=0sec 2024-11-21T01:44:10,952 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:10,952 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:C 2024-11-21T01:44:10,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741898_1074 (size=12301) 2024-11-21T01:44:11,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T01:44:11,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:11,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:11,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:11,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:11,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:11,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:11,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:11,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153511093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:11,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:11,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:11,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:11,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:11,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153511094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:11,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:11,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:11,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:11,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:11,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153511097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:11,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:11,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:11,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:11,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:11,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153511114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:11,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:11,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:11,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:11,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153511117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:11,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:11,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:11,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:11,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T01:44:11,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:11,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:11,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:11,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:11,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:11,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:11,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/690a1385833348d393fbc9138b4d4d74 2024-11-21T01:44:11,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/9f3be1919361425cbc4eca2266e2cb86 is 50, key is test_row_0/C:col10/1732153449747/Put/seqid=0 2024-11-21T01:44:11,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T01:44:11,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:11,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:11,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:11,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:11,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:11,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:11,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741899_1075 (size=12301) 2024-11-21T01:44:11,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T01:44:11,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T01:44:11,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:11,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:11,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:11,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:11,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:11,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:11,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:11,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153511604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:11,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:11,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:11,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:11,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:11,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153511604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:11,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:11,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:11,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:11,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:11,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153511624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:11,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:11,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:11,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:11,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:11,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153511624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:11,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:11,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:11,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:11,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:11,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153511633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:11,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:11,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:11,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:11,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T01:44:11,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:11,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:11,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:11,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:11,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:11,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:11,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/9f3be1919361425cbc4eca2266e2cb86 2024-11-21T01:44:11,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/3a94f45df01d4375b3bcf3b3f8b41a37 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/3a94f45df01d4375b3bcf3b3f8b41a37 2024-11-21T01:44:11,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/3a94f45df01d4375b3bcf3b3f8b41a37, entries=200, sequenceid=281, filesize=14.4 K 2024-11-21T01:44:11,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/690a1385833348d393fbc9138b4d4d74 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/690a1385833348d393fbc9138b4d4d74 2024-11-21T01:44:11,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/690a1385833348d393fbc9138b4d4d74, entries=150, sequenceid=281, filesize=12.0 K 2024-11-21T01:44:11,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/9f3be1919361425cbc4eca2266e2cb86 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/9f3be1919361425cbc4eca2266e2cb86 2024-11-21T01:44:11,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T01:44:11,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:11,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:11,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:11,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:11,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:11,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/9f3be1919361425cbc4eca2266e2cb86, entries=150, sequenceid=281, filesize=12.0 K 2024-11-21T01:44:11,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 06db9be3c4e6682aa8966442e1ef70cc in 1488ms, sequenceid=281, compaction requested=false 2024-11-21T01:44:11,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:12,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T01:44:12,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:12,041 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 06db9be3c4e6682aa8966442e1ef70cc 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T01:44:12,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=A 2024-11-21T01:44:12,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:12,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=B 2024-11-21T01:44:12,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:12,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=C 2024-11-21T01:44:12,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:12,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/1b2c68dca45f414491ea78986d951b08 is 50, key is test_row_0/A:col10/1732153450447/Put/seqid=0 2024-11-21T01:44:12,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741900_1076 (size=12301) 2024-11-21T01:44:12,143 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/1b2c68dca45f414491ea78986d951b08 2024-11-21T01:44:12,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/011b31c4c08a4e99a59d072f94273f4d is 50, key is test_row_0/B:col10/1732153450447/Put/seqid=0 2024-11-21T01:44:12,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741901_1077 (size=12301) 2024-11-21T01:44:12,253 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/011b31c4c08a4e99a59d072f94273f4d 2024-11-21T01:44:12,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/91d9c5c901d543b8a01f016980301597 is 50, key is test_row_0/C:col10/1732153450447/Put/seqid=0 2024-11-21T01:44:12,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741902_1078 (size=12301) 2024-11-21T01:44:12,369 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/91d9c5c901d543b8a01f016980301597 2024-11-21T01:44:12,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/1b2c68dca45f414491ea78986d951b08 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/1b2c68dca45f414491ea78986d951b08 2024-11-21T01:44:12,498 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/1b2c68dca45f414491ea78986d951b08, entries=150, sequenceid=298, filesize=12.0 K 2024-11-21T01:44:12,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/011b31c4c08a4e99a59d072f94273f4d as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/011b31c4c08a4e99a59d072f94273f4d 2024-11-21T01:44:12,574 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/011b31c4c08a4e99a59d072f94273f4d, entries=150, sequenceid=298, filesize=12.0 K 2024-11-21T01:44:12,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/91d9c5c901d543b8a01f016980301597 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/91d9c5c901d543b8a01f016980301597 2024-11-21T01:44:12,597 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/91d9c5c901d543b8a01f016980301597, entries=150, sequenceid=298, filesize=12.0 K 2024-11-21T01:44:12,599 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for 06db9be3c4e6682aa8966442e1ef70cc in 558ms, sequenceid=298, compaction requested=true 2024-11-21T01:44:12,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:12,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:12,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-21T01:44:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-21T01:44:12,616 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-21T01:44:12,616 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2600 sec 2024-11-21T01:44:12,620 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 3.2820 sec 2024-11-21T01:44:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:44:12,666 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 06db9be3c4e6682aa8966442e1ef70cc 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-21T01:44:12,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=A 2024-11-21T01:44:12,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:12,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=B 2024-11-21T01:44:12,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:12,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=C 2024-11-21T01:44:12,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:12,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/6d65d70a5c404824a61234047cd8db3f is 50, key is test_row_0/A:col10/1732153452663/Put/seqid=0 2024-11-21T01:44:12,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741903_1079 (size=17181) 2024-11-21T01:44:12,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732153452737, max timestamp=1732153452753Num uniques 859; 2024-11-21T01:44:12,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732153452737, max timestamp=1732153452753Num uniques 859; 2024-11-21T01:44:12,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T01:44:12,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T01:44:12,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732153452737, max timestamp=1732153452753Num uniques 859; 2024-11-21T01:44:12,765 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T01:44:12,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:12,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153512792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:12,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:12,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:12,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:12,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:12,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153512795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:12,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:12,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:12,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:12,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:12,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153512797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:12,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:12,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153512798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:12,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:12,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:12,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:12,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:12,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:12,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:12,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:12,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153512800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:12,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:12,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:12,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:12,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:12,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153512913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:12,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:12,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:12,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:12,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:12,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153512925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:12,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:12,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:12,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:12,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:12,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153512925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:12,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:12,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153512925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:12,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:12,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:12,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:12,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:12,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:12,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:12,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:12,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153512926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:12,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:12,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:12,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:13,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/6d65d70a5c404824a61234047cd8db3f 2024-11-21T01:44:13,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:13,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153513136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:13,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:13,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153513143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:13,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:13,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:13,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:13,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:13,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153513143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:13,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:13,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153513144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:13,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:13,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153513144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:13,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:13,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:13,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:13,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/bd29be2a9cb84883ac6cc12befa6d133 is 50, key is test_row_0/B:col10/1732153452663/Put/seqid=0 2024-11-21T01:44:13,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741904_1080 (size=12301) 2024-11-21T01:44:13,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:13,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153513454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:13,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:13,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:13,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:13,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:13,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153513454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:13,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:13,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:13,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:13,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:13,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153513463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:13,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:13,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153513463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:13,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:13,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153513464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:13,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T01:44:13,504 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:44:13,504 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:44:13,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-21T01:44:13,507 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:44:13,508 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:44:13,508 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:44:13,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T01:44:13,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T01:44:13,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/bd29be2a9cb84883ac6cc12befa6d133 2024-11-21T01:44:13,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/dac486fa61524c7ca7b0248a78eb22e0 is 50, key is test_row_0/C:col10/1732153452663/Put/seqid=0 2024-11-21T01:44:13,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741905_1081 (size=12301) 2024-11-21T01:44:13,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/dac486fa61524c7ca7b0248a78eb22e0 2024-11-21T01:44:13,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T01:44:13,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:13,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:13,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:13,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:13,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:13,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/6d65d70a5c404824a61234047cd8db3f as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/6d65d70a5c404824a61234047cd8db3f 2024-11-21T01:44:13,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/6d65d70a5c404824a61234047cd8db3f, entries=250, sequenceid=309, filesize=16.8 K 2024-11-21T01:44:13,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/bd29be2a9cb84883ac6cc12befa6d133 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/bd29be2a9cb84883ac6cc12befa6d133 2024-11-21T01:44:13,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/bd29be2a9cb84883ac6cc12befa6d133, entries=150, sequenceid=309, filesize=12.0 K 2024-11-21T01:44:13,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/dac486fa61524c7ca7b0248a78eb22e0 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/dac486fa61524c7ca7b0248a78eb22e0 2024-11-21T01:44:13,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/dac486fa61524c7ca7b0248a78eb22e0, entries=150, sequenceid=309, filesize=12.0 K 2024-11-21T01:44:13,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 06db9be3c4e6682aa8966442e1ef70cc in 1097ms, sequenceid=309, compaction requested=true 2024-11-21T01:44:13,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:13,764 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:44:13,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:44:13,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:13,764 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:44:13,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:44:13,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:13,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:44:13,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:13,774 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57172 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:44:13,774 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/A is initiating minor compaction (all files) 2024-11-21T01:44:13,774 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/A in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:13,774 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/16a83b65d1ec4491a9f768418a6d74f1, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/3a94f45df01d4375b3bcf3b3f8b41a37, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/1b2c68dca45f414491ea78986d951b08, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/6d65d70a5c404824a61234047cd8db3f] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=55.8 K 2024-11-21T01:44:13,775 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:44:13,775 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/B is initiating minor compaction (all files) 2024-11-21T01:44:13,775 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/B in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:13,775 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/54b918c104bf48a3948e22135422f52c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/690a1385833348d393fbc9138b4d4d74, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/011b31c4c08a4e99a59d072f94273f4d, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/bd29be2a9cb84883ac6cc12befa6d133] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=48.7 K 2024-11-21T01:44:13,775 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16a83b65d1ec4491a9f768418a6d74f1, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732153449014 2024-11-21T01:44:13,775 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 54b918c104bf48a3948e22135422f52c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732153449014 2024-11-21T01:44:13,776 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 690a1385833348d393fbc9138b4d4d74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732153449743 2024-11-21T01:44:13,776 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a94f45df01d4375b3bcf3b3f8b41a37, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732153449743 2024-11-21T01:44:13,777 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b2c68dca45f414491ea78986d951b08, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732153450444 2024-11-21T01:44:13,777 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 011b31c4c08a4e99a59d072f94273f4d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732153450444 2024-11-21T01:44:13,777 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d65d70a5c404824a61234047cd8db3f, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732153452640 2024-11-21T01:44:13,778 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting bd29be2a9cb84883ac6cc12befa6d133, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732153452657 2024-11-21T01:44:13,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T01:44:13,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:13,819 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 06db9be3c4e6682aa8966442e1ef70cc 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-21T01:44:13,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=A 2024-11-21T01:44:13,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:13,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=B 2024-11-21T01:44:13,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:13,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=C 2024-11-21T01:44:13,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:13,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T01:44:13,826 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#B#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:13,827 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/488dc2f1fd3946d98d761e75ff9d8049 is 50, key is test_row_0/B:col10/1732153452663/Put/seqid=0 2024-11-21T01:44:13,832 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#A#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:13,833 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/5482f3dd0e3a4171ad4f1be8463fd2d3 is 50, key is test_row_0/A:col10/1732153452663/Put/seqid=0 2024-11-21T01:44:13,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/b1511f787562409a8caad24f68743c7c is 50, key is test_row_0/A:col10/1732153452789/Put/seqid=0 2024-11-21T01:44:13,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741907_1083 (size=12301) 2024-11-21T01:44:13,871 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/b1511f787562409a8caad24f68743c7c 2024-11-21T01:44:13,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741906_1082 (size=13085) 2024-11-21T01:44:13,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741908_1084 (size=13085) 2024-11-21T01:44:13,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/962472780dc849fe87796e9f98d29d40 is 50, key is test_row_0/B:col10/1732153452789/Put/seqid=0 2024-11-21T01:44:13,919 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/5482f3dd0e3a4171ad4f1be8463fd2d3 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/5482f3dd0e3a4171ad4f1be8463fd2d3 2024-11-21T01:44:13,938 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/A of 06db9be3c4e6682aa8966442e1ef70cc into 5482f3dd0e3a4171ad4f1be8463fd2d3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:13,938 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:13,939 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/A, priority=12, startTime=1732153453763; duration=0sec 2024-11-21T01:44:13,939 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:13,939 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:A 2024-11-21T01:44:13,939 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:44:13,941 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:44:13,941 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/C is initiating minor compaction (all files) 2024-11-21T01:44:13,941 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/C in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:13,941 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/001bd063bf1040468625035b1a337510, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/9f3be1919361425cbc4eca2266e2cb86, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/91d9c5c901d543b8a01f016980301597, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/dac486fa61524c7ca7b0248a78eb22e0] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=48.7 K 2024-11-21T01:44:13,942 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 001bd063bf1040468625035b1a337510, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732153449014 2024-11-21T01:44:13,942 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f3be1919361425cbc4eca2266e2cb86, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732153449743 2024-11-21T01:44:13,943 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91d9c5c901d543b8a01f016980301597, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732153450444 2024-11-21T01:44:13,943 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting dac486fa61524c7ca7b0248a78eb22e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732153452657 2024-11-21T01:44:13,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:44:13,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:13,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741909_1085 (size=12301) 2024-11-21T01:44:13,977 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/962472780dc849fe87796e9f98d29d40 2024-11-21T01:44:13,992 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#C#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:13,993 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/c61532d33e3645f3a00d332d62f4bdfe is 50, key is test_row_0/C:col10/1732153452663/Put/seqid=0 2024-11-21T01:44:13,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/69544680ac6f4ac7ad1a46b144c64418 is 50, key is test_row_0/C:col10/1732153452789/Put/seqid=0 2024-11-21T01:44:14,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:14,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153513989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:14,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:14,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:14,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153513989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:14,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:14,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153513989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:14,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:14,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:14,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153513991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:14,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:14,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:14,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:14,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153514001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:14,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:14,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741910_1086 (size=13085) 2024-11-21T01:44:14,061 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/c61532d33e3645f3a00d332d62f4bdfe as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/c61532d33e3645f3a00d332d62f4bdfe 2024-11-21T01:44:14,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741911_1087 (size=12301) 2024-11-21T01:44:14,069 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/C of 06db9be3c4e6682aa8966442e1ef70cc into c61532d33e3645f3a00d332d62f4bdfe(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:14,069 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:14,069 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/C, priority=12, startTime=1732153453764; duration=0sec 2024-11-21T01:44:14,069 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:14,069 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:C 2024-11-21T01:44:14,070 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/69544680ac6f4ac7ad1a46b144c64418 2024-11-21T01:44:14,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/b1511f787562409a8caad24f68743c7c as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/b1511f787562409a8caad24f68743c7c 2024-11-21T01:44:14,101 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/b1511f787562409a8caad24f68743c7c, entries=150, sequenceid=344, filesize=12.0 K 2024-11-21T01:44:14,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/962472780dc849fe87796e9f98d29d40 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/962472780dc849fe87796e9f98d29d40 2024-11-21T01:44:14,114 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/962472780dc849fe87796e9f98d29d40, entries=150, sequenceid=344, filesize=12.0 K 2024-11-21T01:44:14,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/69544680ac6f4ac7ad1a46b144c64418 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/69544680ac6f4ac7ad1a46b144c64418 2024-11-21T01:44:14,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:14,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153514115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:14,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:14,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153514115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:14,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:14,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153514116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:14,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:14,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153514115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:14,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:14,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:14,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153514123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:14,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:14,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:14,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T01:44:14,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:14,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:14,148 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/69544680ac6f4ac7ad1a46b144c64418, entries=150, sequenceid=344, filesize=12.0 K 2024-11-21T01:44:14,149 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 06db9be3c4e6682aa8966442e1ef70cc in 331ms, sequenceid=344, compaction requested=false 2024-11-21T01:44:14,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:14,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:14,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-21T01:44:14,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-21T01:44:14,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-21T01:44:14,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 646 msec 2024-11-21T01:44:14,161 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 654 msec 2024-11-21T01:44:14,296 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/488dc2f1fd3946d98d761e75ff9d8049 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/488dc2f1fd3946d98d761e75ff9d8049 2024-11-21T01:44:14,304 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/B of 06db9be3c4e6682aa8966442e1ef70cc into 488dc2f1fd3946d98d761e75ff9d8049(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:14,304 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:14,304 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/B, priority=12, startTime=1732153453764; duration=0sec 2024-11-21T01:44:14,304 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:14,305 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:B 2024-11-21T01:44:14,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:44:14,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 06db9be3c4e6682aa8966442e1ef70cc 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T01:44:14,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=A 2024-11-21T01:44:14,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:14,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=B 2024-11-21T01:44:14,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:14,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=C 2024-11-21T01:44:14,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:14,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/840c6794d4064519a1cec266925e6782 is 50, key is test_row_0/A:col10/1732153454334/Put/seqid=0 2024-11-21T01:44:14,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:14,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153514369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:14,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:14,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153514369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:14,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:14,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153514371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:14,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:14,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153514375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:14,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:14,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:14,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:14,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:14,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153514386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:14,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:14,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741912_1088 (size=12301) 2024-11-21T01:44:14,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:14,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:14,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153514494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:14,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153514494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:14,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:14,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:14,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153514494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:14,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153514494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:14,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:14,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:14,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153514503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:14,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:14,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T01:44:14,643 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:44:14,644 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:44:14,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-21T01:44:14,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T01:44:14,647 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:44:14,648 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:44:14,648 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:44:14,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:14,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153514703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:14,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:14,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:14,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153514703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:14,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:14,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:14,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153514703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:14,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:14,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153514724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:14,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:14,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153514724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:14,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:14,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:14,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T01:44:14,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T01:44:14,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:14,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:14,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:14,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:14,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:14,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:14,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/840c6794d4064519a1cec266925e6782 2024-11-21T01:44:14,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/08e715696c544f468ee8057b7f67c9c4 is 50, key is test_row_0/B:col10/1732153454334/Put/seqid=0 2024-11-21T01:44:14,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741913_1089 (size=12301) 2024-11-21T01:44:14,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/08e715696c544f468ee8057b7f67c9c4 2024-11-21T01:44:14,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/228374ba78e04dc58afbc52e3ce32b68 is 50, key is test_row_0/C:col10/1732153454334/Put/seqid=0 2024-11-21T01:44:14,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741914_1090 (size=12301) 2024-11-21T01:44:14,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/228374ba78e04dc58afbc52e3ce32b68 2024-11-21T01:44:14,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/840c6794d4064519a1cec266925e6782 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/840c6794d4064519a1cec266925e6782 2024-11-21T01:44:14,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/840c6794d4064519a1cec266925e6782, entries=150, sequenceid=359, filesize=12.0 K 2024-11-21T01:44:14,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/08e715696c544f468ee8057b7f67c9c4 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/08e715696c544f468ee8057b7f67c9c4 2024-11-21T01:44:14,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/08e715696c544f468ee8057b7f67c9c4, entries=150, sequenceid=359, filesize=12.0 K 2024-11-21T01:44:14,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/228374ba78e04dc58afbc52e3ce32b68 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/228374ba78e04dc58afbc52e3ce32b68 2024-11-21T01:44:14,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/228374ba78e04dc58afbc52e3ce32b68, entries=150, sequenceid=359, filesize=12.0 K 2024-11-21T01:44:14,925 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 06db9be3c4e6682aa8966442e1ef70cc in 589ms, sequenceid=359, compaction requested=true 2024-11-21T01:44:14,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:14,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:44:14,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:14,926 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:14,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:44:14,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:14,926 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:14,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:44:14,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:14,938 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:14,938 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/A is initiating minor compaction (all files) 2024-11-21T01:44:14,938 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/A in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:14,939 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/5482f3dd0e3a4171ad4f1be8463fd2d3, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/b1511f787562409a8caad24f68743c7c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/840c6794d4064519a1cec266925e6782] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=36.8 K 2024-11-21T01:44:14,939 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5482f3dd0e3a4171ad4f1be8463fd2d3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732153452657 2024-11-21T01:44:14,944 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:14,944 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/B is initiating minor compaction (all files) 2024-11-21T01:44:14,944 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/B in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:14,944 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/488dc2f1fd3946d98d761e75ff9d8049, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/962472780dc849fe87796e9f98d29d40, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/08e715696c544f468ee8057b7f67c9c4] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=36.8 K 2024-11-21T01:44:14,944 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1511f787562409a8caad24f68743c7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732153452789 2024-11-21T01:44:14,946 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 488dc2f1fd3946d98d761e75ff9d8049, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732153452657 2024-11-21T01:44:14,946 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 840c6794d4064519a1cec266925e6782, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732153453987 2024-11-21T01:44:14,947 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 962472780dc849fe87796e9f98d29d40, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732153452789 2024-11-21T01:44:14,948 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 08e715696c544f468ee8057b7f67c9c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732153453987 2024-11-21T01:44:14,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T01:44:14,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:14,962 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 06db9be3c4e6682aa8966442e1ef70cc 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T01:44:14,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=A 2024-11-21T01:44:14,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:14,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=B 2024-11-21T01:44:14,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:14,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=C 2024-11-21T01:44:14,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:14,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T01:44:14,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/8fd0fe6486af45c9a2745dbc3437f796 is 50, key is test_row_0/A:col10/1732153454382/Put/seqid=0 2024-11-21T01:44:15,016 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#A#compaction#79 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:15,017 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/434cbcfe14f54dfc8b49116bd8e559d2 is 50, key is test_row_0/A:col10/1732153454334/Put/seqid=0 2024-11-21T01:44:15,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:44:15,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:15,021 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#B#compaction#80 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:15,022 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/628c53e431bb46098f8d64ac16d4c149 is 50, key is test_row_0/B:col10/1732153454334/Put/seqid=0 2024-11-21T01:44:15,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741915_1091 (size=12301) 2024-11-21T01:44:15,040 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/8fd0fe6486af45c9a2745dbc3437f796 2024-11-21T01:44:15,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741916_1092 (size=13187) 2024-11-21T01:44:15,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741917_1093 (size=13187) 2024-11-21T01:44:15,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:15,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153515038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:15,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:15,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153515043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:15,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:15,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:15,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153515044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:15,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153515047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:15,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:15,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:15,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153515050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:15,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:15,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:15,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:15,070 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/628c53e431bb46098f8d64ac16d4c149 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/628c53e431bb46098f8d64ac16d4c149 2024-11-21T01:44:15,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/754f3dce7b7d44dc8f73081f2bf9c7c0 is 50, key is test_row_0/B:col10/1732153454382/Put/seqid=0 2024-11-21T01:44:15,078 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/434cbcfe14f54dfc8b49116bd8e559d2 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/434cbcfe14f54dfc8b49116bd8e559d2 2024-11-21T01:44:15,080 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/B of 06db9be3c4e6682aa8966442e1ef70cc into 628c53e431bb46098f8d64ac16d4c149(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:15,080 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:15,080 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/B, priority=13, startTime=1732153454926; duration=0sec 2024-11-21T01:44:15,081 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:15,081 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:B 2024-11-21T01:44:15,081 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:15,084 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:15,084 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/C is initiating minor compaction (all files) 2024-11-21T01:44:15,084 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/C in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:15,084 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/c61532d33e3645f3a00d332d62f4bdfe, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/69544680ac6f4ac7ad1a46b144c64418, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/228374ba78e04dc58afbc52e3ce32b68] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=36.8 K 2024-11-21T01:44:15,085 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting c61532d33e3645f3a00d332d62f4bdfe, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732153452657 2024-11-21T01:44:15,086 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 69544680ac6f4ac7ad1a46b144c64418, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732153452789 2024-11-21T01:44:15,086 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 228374ba78e04dc58afbc52e3ce32b68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732153453987 2024-11-21T01:44:15,087 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/A of 06db9be3c4e6682aa8966442e1ef70cc into 434cbcfe14f54dfc8b49116bd8e559d2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:15,087 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:15,087 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/A, priority=13, startTime=1732153454926; duration=0sec 2024-11-21T01:44:15,088 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:15,088 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:A 2024-11-21T01:44:15,118 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#C#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:15,119 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/13ecd9b1f56c4f96879b19604f10bf00 is 50, key is test_row_0/C:col10/1732153454334/Put/seqid=0 2024-11-21T01:44:15,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741918_1094 (size=12301) 2024-11-21T01:44:15,152 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/754f3dce7b7d44dc8f73081f2bf9c7c0 2024-11-21T01:44:15,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:15,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153515154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:15,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:15,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:15,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153515166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:15,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:15,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153515167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:15,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:15,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:15,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153515169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:15,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:15,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153515169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:15,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:15,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:15,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741919_1095 (size=13187) 2024-11-21T01:44:15,208 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/13ecd9b1f56c4f96879b19604f10bf00 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/13ecd9b1f56c4f96879b19604f10bf00 2024-11-21T01:44:15,216 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/C of 06db9be3c4e6682aa8966442e1ef70cc into 13ecd9b1f56c4f96879b19604f10bf00(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:15,216 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:15,216 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/C, priority=13, startTime=1732153454926; duration=0sec 2024-11-21T01:44:15,217 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:15,217 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:C 2024-11-21T01:44:15,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/27e5ad8dee604086a871ee6e28138d6a is 50, key is test_row_0/C:col10/1732153454382/Put/seqid=0 2024-11-21T01:44:15,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741920_1096 (size=12301) 2024-11-21T01:44:15,235 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/27e5ad8dee604086a871ee6e28138d6a 2024-11-21T01:44:15,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/8fd0fe6486af45c9a2745dbc3437f796 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/8fd0fe6486af45c9a2745dbc3437f796 2024-11-21T01:44:15,252 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/8fd0fe6486af45c9a2745dbc3437f796, entries=150, sequenceid=379, filesize=12.0 K 2024-11-21T01:44:15,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/754f3dce7b7d44dc8f73081f2bf9c7c0 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/754f3dce7b7d44dc8f73081f2bf9c7c0 2024-11-21T01:44:15,271 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/754f3dce7b7d44dc8f73081f2bf9c7c0, entries=150, sequenceid=379, filesize=12.0 K 2024-11-21T01:44:15,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T01:44:15,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/27e5ad8dee604086a871ee6e28138d6a as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/27e5ad8dee604086a871ee6e28138d6a 2024-11-21T01:44:15,291 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/27e5ad8dee604086a871ee6e28138d6a, entries=150, sequenceid=379, filesize=12.0 K 2024-11-21T01:44:15,293 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 06db9be3c4e6682aa8966442e1ef70cc in 331ms, sequenceid=379, compaction requested=false 2024-11-21T01:44:15,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:15,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:15,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-21T01:44:15,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-21T01:44:15,301 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-21T01:44:15,301 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 650 msec 2024-11-21T01:44:15,304 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 658 msec 2024-11-21T01:44:15,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:44:15,370 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 06db9be3c4e6682aa8966442e1ef70cc 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T01:44:15,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=A 2024-11-21T01:44:15,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:15,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=B 2024-11-21T01:44:15,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:15,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=C 2024-11-21T01:44:15,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:15,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/e3768d7a49574375a057af36b48c6baf is 50, key is test_row_0/A:col10/1732153455045/Put/seqid=0 2024-11-21T01:44:15,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741921_1097 (size=12301) 2024-11-21T01:44:15,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:15,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153515431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:15,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:15,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153515432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:15,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:15,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153515443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:15,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:15,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:15,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153515447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:15,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:15,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:15,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:15,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153515447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:15,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:15,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/e3768d7a49574375a057af36b48c6baf 2024-11-21T01:44:15,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:15,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/f8789e022b9645e693c571071ffb060d is 50, key is test_row_0/B:col10/1732153455045/Put/seqid=0 2024-11-21T01:44:15,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741922_1098 (size=12301) 2024-11-21T01:44:15,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:15,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153515555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:15,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:15,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:15,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153515564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:15,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:15,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:15,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153515564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:15,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153515564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:15,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:15,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:15,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153515565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:15,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:15,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:15,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:15,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:15,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153515773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:15,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:15,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T01:44:15,786 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:44:15,786 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:44:15,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-21T01:44:15,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T01:44:15,788 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:44:15,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:15,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153515784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:15,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:15,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153515784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:15,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:15,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153515784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:15,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153515784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:15,793 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:44:15,793 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:44:15,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T01:44:15,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/f8789e022b9645e693c571071ffb060d 2024-11-21T01:44:15,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/8cb3543717f74efba2b195487b349285 is 50, key is test_row_0/C:col10/1732153455045/Put/seqid=0 2024-11-21T01:44:15,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T01:44:15,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:15,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:15,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:15,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:15,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:15,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:15,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741923_1099 (size=12301) 2024-11-21T01:44:16,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:16,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153516083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:16,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:16,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:16,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153516093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:16,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:16,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:16,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153516093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:16,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:16,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T01:44:16,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:16,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:16,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:16,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:16,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:16,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:16,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T01:44:16,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:16,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153516104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:16,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:16,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:16,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153516104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:16,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:16,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T01:44:16,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:16,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:16,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:16,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:16,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:16,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:16,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/8cb3543717f74efba2b195487b349285 2024-11-21T01:44:16,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/e3768d7a49574375a057af36b48c6baf as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/e3768d7a49574375a057af36b48c6baf 2024-11-21T01:44:16,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/e3768d7a49574375a057af36b48c6baf, entries=150, sequenceid=395, filesize=12.0 K 2024-11-21T01:44:16,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/f8789e022b9645e693c571071ffb060d as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/f8789e022b9645e693c571071ffb060d 2024-11-21T01:44:16,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/f8789e022b9645e693c571071ffb060d, entries=150, sequenceid=395, filesize=12.0 K 2024-11-21T01:44:16,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/8cb3543717f74efba2b195487b349285 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/8cb3543717f74efba2b195487b349285 2024-11-21T01:44:16,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/8cb3543717f74efba2b195487b349285, entries=150, sequenceid=395, filesize=12.0 K 2024-11-21T01:44:16,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 06db9be3c4e6682aa8966442e1ef70cc in 1028ms, sequenceid=395, compaction requested=true 2024-11-21T01:44:16,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:16,398 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:16,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:44:16,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:16,399 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:16,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:44:16,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:16,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:44:16,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:16,405 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:16,405 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/B is initiating minor compaction (all files) 2024-11-21T01:44:16,405 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/B in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:16,405 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/628c53e431bb46098f8d64ac16d4c149, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/754f3dce7b7d44dc8f73081f2bf9c7c0, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/f8789e022b9645e693c571071ffb060d] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=36.9 K 2024-11-21T01:44:16,405 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:16,405 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/A is initiating minor compaction (all files) 2024-11-21T01:44:16,406 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/A in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:16,406 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/434cbcfe14f54dfc8b49116bd8e559d2, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/8fd0fe6486af45c9a2745dbc3437f796, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/e3768d7a49574375a057af36b48c6baf] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=36.9 K 2024-11-21T01:44:16,406 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 628c53e431bb46098f8d64ac16d4c149, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732153453987 2024-11-21T01:44:16,407 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 434cbcfe14f54dfc8b49116bd8e559d2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732153453987 2024-11-21T01:44:16,407 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 754f3dce7b7d44dc8f73081f2bf9c7c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732153454365 2024-11-21T01:44:16,408 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fd0fe6486af45c9a2745dbc3437f796, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732153454365 2024-11-21T01:44:16,408 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting f8789e022b9645e693c571071ffb060d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732153455020 2024-11-21T01:44:16,408 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3768d7a49574375a057af36b48c6baf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732153455020 2024-11-21T01:44:16,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T01:44:16,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:16,412 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 06db9be3c4e6682aa8966442e1ef70cc 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T01:44:16,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=A 2024-11-21T01:44:16,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:16,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=B 2024-11-21T01:44:16,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:16,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=C 2024-11-21T01:44:16,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:16,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T01:44:16,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/b38fec445b7f4583b2d58ac10fa991a3 is 50, key is test_row_0/A:col10/1732153455442/Put/seqid=0 2024-11-21T01:44:16,437 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#B#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:16,438 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/e8a617c03f324a2283ff1ecb714c85c3 is 50, key is test_row_0/B:col10/1732153455045/Put/seqid=0 2024-11-21T01:44:16,448 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#A#compaction#89 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:16,449 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/8731da3435a748559103ab8eba382928 is 50, key is test_row_0/A:col10/1732153455045/Put/seqid=0 2024-11-21T01:44:16,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741925_1101 (size=13289) 2024-11-21T01:44:16,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741924_1100 (size=12301) 2024-11-21T01:44:16,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741926_1102 (size=13289) 2024-11-21T01:44:16,481 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/b38fec445b7f4583b2d58ac10fa991a3 2024-11-21T01:44:16,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/f441de8ae21541aebcce5a3037a09867 is 50, key is test_row_0/B:col10/1732153455442/Put/seqid=0 2024-11-21T01:44:16,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741927_1103 (size=12301) 2024-11-21T01:44:16,506 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/f441de8ae21541aebcce5a3037a09867 2024-11-21T01:44:16,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/7af288d1e7f944f68f9ea4c6e0527ba7 is 50, key is test_row_0/C:col10/1732153455442/Put/seqid=0 2024-11-21T01:44:16,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741928_1104 (size=12301) 2024-11-21T01:44:16,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:16,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:44:16,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:16,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153516618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:16,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:16,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:16,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153516620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:16,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:16,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:16,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153516620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:16,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:16,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153516622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:16,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153516622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:16,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:16,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153516733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:16,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:16,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153516733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:16,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:16,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153516734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:16,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:16,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153516734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:16,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:16,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153516734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:16,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:16,892 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/8731da3435a748559103ab8eba382928 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/8731da3435a748559103ab8eba382928 2024-11-21T01:44:16,893 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/e8a617c03f324a2283ff1ecb714c85c3 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/e8a617c03f324a2283ff1ecb714c85c3 2024-11-21T01:44:16,904 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/B of 06db9be3c4e6682aa8966442e1ef70cc into e8a617c03f324a2283ff1ecb714c85c3(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:16,904 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:16,904 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/B, priority=13, startTime=1732153456399; duration=0sec 2024-11-21T01:44:16,904 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:16,904 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:B 2024-11-21T01:44:16,904 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:16,906 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/A of 06db9be3c4e6682aa8966442e1ef70cc into 8731da3435a748559103ab8eba382928(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:16,906 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:16,906 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/A, priority=13, startTime=1732153456398; duration=0sec 2024-11-21T01:44:16,906 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:16,906 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:A 2024-11-21T01:44:16,907 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:16,907 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/C is initiating minor compaction (all files) 2024-11-21T01:44:16,907 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/C in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:16,908 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/13ecd9b1f56c4f96879b19604f10bf00, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/27e5ad8dee604086a871ee6e28138d6a, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/8cb3543717f74efba2b195487b349285] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=36.9 K 2024-11-21T01:44:16,908 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 13ecd9b1f56c4f96879b19604f10bf00, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732153453987 2024-11-21T01:44:16,909 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 27e5ad8dee604086a871ee6e28138d6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732153454365 2024-11-21T01:44:16,909 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cb3543717f74efba2b195487b349285, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732153455020 2024-11-21T01:44:16,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T01:44:16,927 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#C#compaction#92 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:16,928 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/67c7227340c640498295bff4ed7cf364 is 50, key is test_row_0/C:col10/1732153455045/Put/seqid=0 2024-11-21T01:44:16,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:16,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153516943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:16,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:16,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741929_1105 (size=13289) 2024-11-21T01:44:16,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:16,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153516954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:16,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:16,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:16,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153516954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:16,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:16,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153516955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:16,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:16,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:16,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153516955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:16,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:16,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:16,967 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/67c7227340c640498295bff4ed7cf364 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/67c7227340c640498295bff4ed7cf364 2024-11-21T01:44:16,973 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/7af288d1e7f944f68f9ea4c6e0527ba7 2024-11-21T01:44:16,982 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/C of 06db9be3c4e6682aa8966442e1ef70cc into 67c7227340c640498295bff4ed7cf364(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:16,982 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:16,983 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/C, priority=13, startTime=1732153456400; duration=0sec 2024-11-21T01:44:16,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/b38fec445b7f4583b2d58ac10fa991a3 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/b38fec445b7f4583b2d58ac10fa991a3 2024-11-21T01:44:16,983 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:16,985 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:C 2024-11-21T01:44:16,989 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/b38fec445b7f4583b2d58ac10fa991a3, entries=150, sequenceid=415, filesize=12.0 K 2024-11-21T01:44:16,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/f441de8ae21541aebcce5a3037a09867 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/f441de8ae21541aebcce5a3037a09867 2024-11-21T01:44:16,995 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/f441de8ae21541aebcce5a3037a09867, entries=150, sequenceid=415, filesize=12.0 K 2024-11-21T01:44:16,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/7af288d1e7f944f68f9ea4c6e0527ba7 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/7af288d1e7f944f68f9ea4c6e0527ba7 2024-11-21T01:44:17,003 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/7af288d1e7f944f68f9ea4c6e0527ba7, entries=150, sequenceid=415, filesize=12.0 K 2024-11-21T01:44:17,004 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 06db9be3c4e6682aa8966442e1ef70cc in 592ms, sequenceid=415, compaction requested=false 2024-11-21T01:44:17,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:17,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:17,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-21T01:44:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-21T01:44:17,008 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-21T01:44:17,008 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2130 sec 2024-11-21T01:44:17,011 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.2230 sec 2024-11-21T01:44:17,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:44:17,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 06db9be3c4e6682aa8966442e1ef70cc 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T01:44:17,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=A 2024-11-21T01:44:17,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:17,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=B 2024-11-21T01:44:17,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:17,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=C 2024-11-21T01:44:17,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:17,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/3a2262f319514517863af260324ad5f5 is 50, key is test_row_0/A:col10/1732153456619/Put/seqid=0 2024-11-21T01:44:17,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741930_1106 (size=14741) 2024-11-21T01:44:17,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:17,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153517297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:17,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:17,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:17,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153517305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:17,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:17,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153517306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:17,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:17,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:17,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:17,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153517307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:17,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:17,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:17,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153517314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:17,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:17,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:17,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153517413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:17,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:17,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:17,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153517423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:17,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:17,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153517424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:17,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:17,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:17,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:17,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153517424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:17,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:17,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:17,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153517424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:17,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:17,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:17,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153517622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:17,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:17,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:17,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153517632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:17,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:17,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:17,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153517633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:17,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:17,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153517633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:17,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:17,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153517633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:17,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/3a2262f319514517863af260324ad5f5 2024-11-21T01:44:17,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/5cb6e55a6b72452ba0702bac00c0ad2b is 50, key is test_row_0/B:col10/1732153456619/Put/seqid=0 2024-11-21T01:44:17,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741931_1107 (size=12301) 2024-11-21T01:44:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T01:44:17,933 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:44:17,934 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:44:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-21T01:44:17,937 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:44:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T01:44:17,938 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:44:17,938 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:44:17,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:17,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153517933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:17,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:17,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153517943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:17,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:17,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:17,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153517953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:17,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:17,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:17,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153517961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153517961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:17,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:17,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:17,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T01:44:18,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T01:44:18,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:18,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:18,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:18,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:18,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:18,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/5cb6e55a6b72452ba0702bac00c0ad2b 2024-11-21T01:44:18,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/0303c49b8a40456a81764d0cb7eb42c9 is 50, key is test_row_0/C:col10/1732153456619/Put/seqid=0 2024-11-21T01:44:18,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741932_1108 (size=12301) 2024-11-21T01:44:18,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/0303c49b8a40456a81764d0cb7eb42c9 2024-11-21T01:44:18,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/3a2262f319514517863af260324ad5f5 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/3a2262f319514517863af260324ad5f5 2024-11-21T01:44:18,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/3a2262f319514517863af260324ad5f5, entries=200, sequenceid=432, filesize=14.4 K 2024-11-21T01:44:18,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/5cb6e55a6b72452ba0702bac00c0ad2b as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/5cb6e55a6b72452ba0702bac00c0ad2b 2024-11-21T01:44:18,190 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/5cb6e55a6b72452ba0702bac00c0ad2b, entries=150, sequenceid=432, filesize=12.0 K 2024-11-21T01:44:18,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/0303c49b8a40456a81764d0cb7eb42c9 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/0303c49b8a40456a81764d0cb7eb42c9 2024-11-21T01:44:18,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/0303c49b8a40456a81764d0cb7eb42c9, entries=150, sequenceid=432, filesize=12.0 K 2024-11-21T01:44:18,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 06db9be3c4e6682aa8966442e1ef70cc in 942ms, sequenceid=432, compaction requested=true 2024-11-21T01:44:18,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:18,200 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:18,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:44:18,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:18,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:44:18,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:18,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:44:18,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T01:44:18,202 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:18,203 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:18,203 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/A is initiating minor compaction (all files) 2024-11-21T01:44:18,203 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/A in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:18,203 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/8731da3435a748559103ab8eba382928, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/b38fec445b7f4583b2d58ac10fa991a3, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/3a2262f319514517863af260324ad5f5] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=39.4 K 2024-11-21T01:44:18,204 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:18,204 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/B is initiating minor compaction (all files) 2024-11-21T01:44:18,204 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8731da3435a748559103ab8eba382928, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732153455020 2024-11-21T01:44:18,204 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/B in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:18,204 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/e8a617c03f324a2283ff1ecb714c85c3, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/f441de8ae21541aebcce5a3037a09867, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/5cb6e55a6b72452ba0702bac00c0ad2b] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=37.0 K 2024-11-21T01:44:18,205 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting b38fec445b7f4583b2d58ac10fa991a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732153455405 2024-11-21T01:44:18,205 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting e8a617c03f324a2283ff1ecb714c85c3, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732153455020 2024-11-21T01:44:18,205 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting f441de8ae21541aebcce5a3037a09867, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732153455405 2024-11-21T01:44:18,205 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a2262f319514517863af260324ad5f5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1732153456611 2024-11-21T01:44:18,205 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cb6e55a6b72452ba0702bac00c0ad2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1732153456617 2024-11-21T01:44:18,222 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#B#compaction#96 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:18,223 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/560347b4a6a24f2ebb9c3c5a87191155 is 50, key is test_row_0/B:col10/1732153456619/Put/seqid=0 2024-11-21T01:44:18,224 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#A#compaction#97 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:18,225 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/993e53ddd8c145febc3fd27d1faf41b0 is 50, key is test_row_0/A:col10/1732153456619/Put/seqid=0 2024-11-21T01:44:18,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741933_1109 (size=13391) 2024-11-21T01:44:18,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T01:44:18,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:18,244 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 06db9be3c4e6682aa8966442e1ef70cc 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T01:44:18,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=A 2024-11-21T01:44:18,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:18,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=B 2024-11-21T01:44:18,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:18,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=C 2024-11-21T01:44:18,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T01:44:18,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741934_1110 (size=13391) 2024-11-21T01:44:18,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/1b02005318bc44e38b22fec60fc4cdd7 is 50, key is test_row_0/A:col10/1732153457294/Put/seqid=0 2024-11-21T01:44:18,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741935_1111 (size=12301) 2024-11-21T01:44:18,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:44:18,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:18,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:18,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153518479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:18,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:18,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:18,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:18,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:18,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153518480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:18,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:18,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:18,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:18,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:18,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153518481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:18,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:18,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:18,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:18,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:18,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153518482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:18,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:18,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:18,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:18,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:18,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153518486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:18,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:18,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:18,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:18,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T01:44:18,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:18,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153518586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:18,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:18,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:18,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:18,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:18,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153518585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:18,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:18,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:18,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:18,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153518593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:18,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:18,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153518594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:18,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:18,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:18,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:18,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:18,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:18,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:18,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:18,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153518595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:18,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:18,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:18,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:18,665 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/560347b4a6a24f2ebb9c3c5a87191155 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/560347b4a6a24f2ebb9c3c5a87191155 2024-11-21T01:44:18,683 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/993e53ddd8c145febc3fd27d1faf41b0 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/993e53ddd8c145febc3fd27d1faf41b0 2024-11-21T01:44:18,686 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/B of 06db9be3c4e6682aa8966442e1ef70cc into 560347b4a6a24f2ebb9c3c5a87191155(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:18,686 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:18,686 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/B, priority=13, startTime=1732153458200; duration=0sec 2024-11-21T01:44:18,689 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:18,689 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:B 2024-11-21T01:44:18,689 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:18,695 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:18,695 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/C is initiating minor compaction (all files) 2024-11-21T01:44:18,695 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/C in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:18,696 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/A of 06db9be3c4e6682aa8966442e1ef70cc into 993e53ddd8c145febc3fd27d1faf41b0(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:18,696 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:18,696 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/A, priority=13, startTime=1732153458200; duration=0sec 2024-11-21T01:44:18,696 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/67c7227340c640498295bff4ed7cf364, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/7af288d1e7f944f68f9ea4c6e0527ba7, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/0303c49b8a40456a81764d0cb7eb42c9] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=37.0 K 2024-11-21T01:44:18,696 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:18,696 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:A 2024-11-21T01:44:18,696 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 67c7227340c640498295bff4ed7cf364, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732153455020 2024-11-21T01:44:18,697 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 7af288d1e7f944f68f9ea4c6e0527ba7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732153455405 2024-11-21T01:44:18,697 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 0303c49b8a40456a81764d0cb7eb42c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1732153456617 2024-11-21T01:44:18,697 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/1b02005318bc44e38b22fec60fc4cdd7 2024-11-21T01:44:18,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/e08bc71f6a984dfc997e7fd9e39ad5e0 is 50, key is test_row_0/B:col10/1732153457294/Put/seqid=0 2024-11-21T01:44:18,737 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#C#compaction#100 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:18,738 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/8cc8e2b55df44e768589e8d15edec034 is 50, key is test_row_0/C:col10/1732153456619/Put/seqid=0 2024-11-21T01:44:18,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741936_1112 (size=12301) 2024-11-21T01:44:18,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:18,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153518804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:18,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741937_1113 (size=13391) 2024-11-21T01:44:18,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:18,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:18,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:18,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153518804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:18,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:18,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:18,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:18,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153518829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:18,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153518831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:18,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153518831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:18,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:18,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:18,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:19,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T01:44:19,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153519115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:19,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153519115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:19,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:19,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153519143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:19,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:19,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153519146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:19,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:19,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153519146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:19,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:19,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:19,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:19,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:19,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:19,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:19,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:19,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:19,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:19,195 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/e08bc71f6a984dfc997e7fd9e39ad5e0 2024-11-21T01:44:19,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/4be627756b614fe196897c1501a094bc is 50, key is test_row_0/C:col10/1732153457294/Put/seqid=0 2024-11-21T01:44:19,212 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/8cc8e2b55df44e768589e8d15edec034 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/8cc8e2b55df44e768589e8d15edec034 2024-11-21T01:44:19,219 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/C of 06db9be3c4e6682aa8966442e1ef70cc into 8cc8e2b55df44e768589e8d15edec034(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:19,219 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:19,219 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/C, priority=13, startTime=1732153458200; duration=0sec 2024-11-21T01:44:19,219 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:19,219 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:C 2024-11-21T01:44:19,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741938_1114 (size=12301) 2024-11-21T01:44:19,232 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/4be627756b614fe196897c1501a094bc 2024-11-21T01:44:19,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/1b02005318bc44e38b22fec60fc4cdd7 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/1b02005318bc44e38b22fec60fc4cdd7 2024-11-21T01:44:19,260 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/1b02005318bc44e38b22fec60fc4cdd7, entries=150, sequenceid=451, filesize=12.0 K 2024-11-21T01:44:19,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/e08bc71f6a984dfc997e7fd9e39ad5e0 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/e08bc71f6a984dfc997e7fd9e39ad5e0 2024-11-21T01:44:19,277 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/e08bc71f6a984dfc997e7fd9e39ad5e0, entries=150, sequenceid=451, filesize=12.0 K 2024-11-21T01:44:19,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/4be627756b614fe196897c1501a094bc as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/4be627756b614fe196897c1501a094bc 2024-11-21T01:44:19,284 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/4be627756b614fe196897c1501a094bc, entries=150, sequenceid=451, filesize=12.0 K 2024-11-21T01:44:19,286 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 06db9be3c4e6682aa8966442e1ef70cc in 1041ms, sequenceid=451, compaction requested=false 2024-11-21T01:44:19,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:19,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:19,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-21T01:44:19,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-21T01:44:19,290 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-21T01:44:19,290 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3500 sec 2024-11-21T01:44:19,293 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.3570 sec 2024-11-21T01:44:19,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:44:19,626 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 06db9be3c4e6682aa8966442e1ef70cc 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-21T01:44:19,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=A 2024-11-21T01:44:19,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:19,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=B 2024-11-21T01:44:19,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:19,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=C 2024-11-21T01:44:19,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:19,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/5fe1308046e748c4b9a20b5279fee2d3 is 50, key is test_row_0/A:col10/1732153458456/Put/seqid=0 2024-11-21T01:44:19,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741939_1115 (size=14741) 2024-11-21T01:44:19,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/5fe1308046e748c4b9a20b5279fee2d3 2024-11-21T01:44:19,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:19,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58186 deadline: 1732153519663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:19,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:19,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:19,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:19,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/63d8ca6089e04dd29880d6512c540c01 is 50, key is test_row_0/B:col10/1732153458456/Put/seqid=0 2024-11-21T01:44:19,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:19,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153519669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:19,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:19,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:19,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:19,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153519669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:19,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:19,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:19,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:19,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:19,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:19,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153519671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:19,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:19,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:19,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:19,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:19,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153519675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:19,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:19,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:19,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:19,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741940_1116 (size=12301) 2024-11-21T01:44:19,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:19,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153519793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:19,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:19,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:19,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:19,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:19,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153519794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:19,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:19,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:19,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:19,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153519794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:19,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:19,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:19,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:19,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:19,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:19,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153519802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:19,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:19,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:19,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:20,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:20,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153520002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:20,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:20,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:20,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:20,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:20,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153520013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:20,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:20,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153520013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:20,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:20,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:20,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:20,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:20,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:20,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:20,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153520013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:20,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:20,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:20,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:20,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:20,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T01:44:20,084 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:44:20,084 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:44:20,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-21T01:44:20,087 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:44:20,088 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:44:20,088 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:44:20,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T01:44:20,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/63d8ca6089e04dd29880d6512c540c01 2024-11-21T01:44:20,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/7a4c990255394730b965c649646563c7 is 50, key is test_row_0/C:col10/1732153458456/Put/seqid=0 2024-11-21T01:44:20,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741941_1117 (size=12301) 2024-11-21T01:44:20,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T01:44:20,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-21T01:44:20,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:20,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:20,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:20,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:20,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:20,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:20,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:20,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58118 deadline: 1732153520313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:20,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:20,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:20,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:20,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:20,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58150 deadline: 1732153520323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:20,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58176 deadline: 1732153520324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:20,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:20,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:20,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:20,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:20,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:20,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:20,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:20,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58136 deadline: 1732153520325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:20,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:20,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:20,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:20,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-21T01:44:20,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:20,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:20,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:20,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:20,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:20,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T01:44:20,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/7a4c990255394730b965c649646563c7 2024-11-21T01:44:20,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-21T01:44:20,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:20,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:20,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:20,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:20,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:20,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:20,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/5fe1308046e748c4b9a20b5279fee2d3 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/5fe1308046e748c4b9a20b5279fee2d3 2024-11-21T01:44:20,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/5fe1308046e748c4b9a20b5279fee2d3, entries=200, sequenceid=470, filesize=14.4 K 2024-11-21T01:44:20,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/63d8ca6089e04dd29880d6512c540c01 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/63d8ca6089e04dd29880d6512c540c01 2024-11-21T01:44:20,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/63d8ca6089e04dd29880d6512c540c01, entries=150, sequenceid=470, filesize=12.0 K 2024-11-21T01:44:20,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/7a4c990255394730b965c649646563c7 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/7a4c990255394730b965c649646563c7 2024-11-21T01:44:20,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/7a4c990255394730b965c649646563c7, entries=150, sequenceid=470, filesize=12.0 K 2024-11-21T01:44:20,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 06db9be3c4e6682aa8966442e1ef70cc in 977ms, sequenceid=470, compaction requested=true 2024-11-21T01:44:20,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:20,603 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:20,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:44:20,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:20,604 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:20,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:44:20,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:20,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 06db9be3c4e6682aa8966442e1ef70cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:44:20,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:20,605 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:20,605 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/A is initiating minor compaction (all files) 2024-11-21T01:44:20,606 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/A in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:20,606 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/993e53ddd8c145febc3fd27d1faf41b0, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/1b02005318bc44e38b22fec60fc4cdd7, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/5fe1308046e748c4b9a20b5279fee2d3] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=39.5 K 2024-11-21T01:44:20,606 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 993e53ddd8c145febc3fd27d1faf41b0, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1732153456617 2024-11-21T01:44:20,607 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:20,607 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b02005318bc44e38b22fec60fc4cdd7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1732153457294 2024-11-21T01:44:20,607 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/B is initiating minor compaction (all files) 2024-11-21T01:44:20,607 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/B in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:20,607 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/560347b4a6a24f2ebb9c3c5a87191155, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/e08bc71f6a984dfc997e7fd9e39ad5e0, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/63d8ca6089e04dd29880d6512c540c01] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=37.1 K 2024-11-21T01:44:20,609 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fe1308046e748c4b9a20b5279fee2d3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1732153458456 2024-11-21T01:44:20,609 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 560347b4a6a24f2ebb9c3c5a87191155, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1732153456617 2024-11-21T01:44:20,613 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting e08bc71f6a984dfc997e7fd9e39ad5e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1732153457294 2024-11-21T01:44:20,614 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 63d8ca6089e04dd29880d6512c540c01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1732153458456 2024-11-21T01:44:20,632 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#A#compaction#105 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:20,632 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/dee0875a793d4d6cb79d7cea1e8ef240 is 50, key is test_row_0/A:col10/1732153458456/Put/seqid=0 2024-11-21T01:44:20,636 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-21T01:44:20,636 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:44:20,644 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-21T01:44:20,644 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:44:20,645 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-21T01:44:20,645 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:44:20,646 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-21T01:44:20,646 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:44:20,647 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#B#compaction#106 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:20,648 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/97eac630e72d476580de677af281ca56 is 50, key is test_row_0/B:col10/1732153458456/Put/seqid=0 2024-11-21T01:44:20,648 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:20,648 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:20,648 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:20,651 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:20,651 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:20,652 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:44:20,652 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:44:20,652 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:20,652 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:44:20,653 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:20,654 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:20,654 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:44:20,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741942_1118 (size=13493) 2024-11-21T01:44:20,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741943_1119 (size=13493) 2024-11-21T01:44:20,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:44:20,697 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 06db9be3c4e6682aa8966442e1ef70cc 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-21T01:44:20,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=A 2024-11-21T01:44:20,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:20,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=B 2024-11-21T01:44:20,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:20,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=C 2024-11-21T01:44:20,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:20,698 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-21T01:44:20,698 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:44:20,698 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:20,698 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:20,698 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:44:20,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/fe90cc7e11874d0f8c882b8749f0a9f4 is 50, key is test_row_0/A:col10/1732153459657/Put/seqid=0 2024-11-21T01:44:20,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741944_1120 (size=12301) 2024-11-21T01:44:20,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T01:44:20,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-21T01:44:20,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:20,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:20,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:20,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:20,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:20,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:20,824 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-21T01:44:20,824 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:44:20,824 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:20,824 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:20,824 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:44:20,834 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-21T01:44:20,834 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:44:20,834 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:20,834 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:20,834 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:44:20,835 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-21T01:44:20,835 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:44:20,835 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:20,835 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:20,835 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:44:20,845 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-21T01:44:20,845 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:44:20,845 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:20,845 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:20,845 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:44:20,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-21T01:44:20,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:20,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:20,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:20,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:20,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:20,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:21,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-21T01:44:21,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:21,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:21,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:21,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:21,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:21,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:21,069 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/dee0875a793d4d6cb79d7cea1e8ef240 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/dee0875a793d4d6cb79d7cea1e8ef240 2024-11-21T01:44:21,075 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/A of 06db9be3c4e6682aa8966442e1ef70cc into dee0875a793d4d6cb79d7cea1e8ef240(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:21,075 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:21,075 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/97eac630e72d476580de677af281ca56 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/97eac630e72d476580de677af281ca56 2024-11-21T01:44:21,075 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/A, priority=13, startTime=1732153460603; duration=0sec 2024-11-21T01:44:21,075 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:21,075 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:A 2024-11-21T01:44:21,075 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:21,076 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:21,076 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 06db9be3c4e6682aa8966442e1ef70cc/C is initiating minor compaction (all files) 2024-11-21T01:44:21,076 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 06db9be3c4e6682aa8966442e1ef70cc/C in TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:21,076 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/8cc8e2b55df44e768589e8d15edec034, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/4be627756b614fe196897c1501a094bc, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/7a4c990255394730b965c649646563c7] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp, totalSize=37.1 K 2024-11-21T01:44:21,077 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cc8e2b55df44e768589e8d15edec034, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1732153456617 2024-11-21T01:44:21,077 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4be627756b614fe196897c1501a094bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1732153457294 2024-11-21T01:44:21,078 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a4c990255394730b965c649646563c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1732153458456 2024-11-21T01:44:21,081 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/B of 06db9be3c4e6682aa8966442e1ef70cc into 97eac630e72d476580de677af281ca56(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:21,081 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:21,081 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/B, priority=13, startTime=1732153460604; duration=0sec 2024-11-21T01:44:21,081 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:21,081 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:B 2024-11-21T01:44:21,087 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 06db9be3c4e6682aa8966442e1ef70cc#C#compaction#108 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:21,087 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/779417a26c5045dabfc44a8b6ace6691 is 50, key is test_row_0/C:col10/1732153458456/Put/seqid=0 2024-11-21T01:44:21,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741945_1121 (size=13493) 2024-11-21T01:44:21,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/fe90cc7e11874d0f8c882b8749f0a9f4 2024-11-21T01:44:21,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/0608828926634165a01916cb7a0d5782 is 50, key is test_row_0/B:col10/1732153459657/Put/seqid=0 2024-11-21T01:44:21,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741946_1122 (size=12301) 2024-11-21T01:44:21,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-21T01:44:21,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:21,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:21,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:21,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:21,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:21,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:21,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T01:44:21,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-21T01:44:21,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:21,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:21,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:21,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:21,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:21,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:21,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-21T01:44:21,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:21,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. as already flushing 2024-11-21T01:44:21,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:21,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:21,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:21,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:21,498 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/779417a26c5045dabfc44a8b6ace6691 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/779417a26c5045dabfc44a8b6ace6691 2024-11-21T01:44:21,503 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 06db9be3c4e6682aa8966442e1ef70cc/C of 06db9be3c4e6682aa8966442e1ef70cc into 779417a26c5045dabfc44a8b6ace6691(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:21,503 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:21,503 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc., storeName=06db9be3c4e6682aa8966442e1ef70cc/C, priority=13, startTime=1732153460604; duration=0sec 2024-11-21T01:44:21,503 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:21,503 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 06db9be3c4e6682aa8966442e1ef70cc:C 2024-11-21T01:44:21,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/0608828926634165a01916cb7a0d5782 2024-11-21T01:44:21,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/75d768714fb6429cacda071a8a120af8 is 50, key is test_row_0/C:col10/1732153459657/Put/seqid=0 2024-11-21T01:44:21,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741947_1123 (size=12301) 2024-11-21T01:44:21,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/75d768714fb6429cacda071a8a120af8 2024-11-21T01:44:21,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/fe90cc7e11874d0f8c882b8749f0a9f4 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/fe90cc7e11874d0f8c882b8749f0a9f4 2024-11-21T01:44:21,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/fe90cc7e11874d0f8c882b8749f0a9f4, entries=150, sequenceid=487, filesize=12.0 K 2024-11-21T01:44:21,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/0608828926634165a01916cb7a0d5782 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/0608828926634165a01916cb7a0d5782 2024-11-21T01:44:21,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/0608828926634165a01916cb7a0d5782, entries=150, sequenceid=487, filesize=12.0 K 2024-11-21T01:44:21,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/75d768714fb6429cacda071a8a120af8 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/75d768714fb6429cacda071a8a120af8 2024-11-21T01:44:21,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/75d768714fb6429cacda071a8a120af8, entries=150, sequenceid=487, filesize=12.0 K 2024-11-21T01:44:21,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=26.84 KB/27480 for 06db9be3c4e6682aa8966442e1ef70cc in 860ms, sequenceid=487, compaction requested=false 2024-11-21T01:44:21,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:21,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-21T01:44:21,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:21,639 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 06db9be3c4e6682aa8966442e1ef70cc 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-21T01:44:21,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=A 2024-11-21T01:44:21,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:21,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=B 2024-11-21T01:44:21,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:21,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 06db9be3c4e6682aa8966442e1ef70cc, store=C 2024-11-21T01:44:21,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:21,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/761482d657d8406abeb82a606c330b83 is 50, key is test_row_0/A:col10/1732153460843/Put/seqid=0 2024-11-21T01:44:21,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741948_1124 (size=12301) 2024-11-21T01:44:21,650 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/761482d657d8406abeb82a606c330b83 2024-11-21T01:44:21,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/4d08772d9cf641bba855d0af3c82f6b3 is 50, key is test_row_0/B:col10/1732153460843/Put/seqid=0 2024-11-21T01:44:21,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741949_1125 (size=12301) 2024-11-21T01:44:22,064 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/4d08772d9cf641bba855d0af3c82f6b3 2024-11-21T01:44:22,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/12b42db510ab4e9695bc691980f2c5fd is 50, key is test_row_0/C:col10/1732153460843/Put/seqid=0 2024-11-21T01:44:22,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741950_1126 (size=12301) 2024-11-21T01:44:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T01:44:22,478 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/12b42db510ab4e9695bc691980f2c5fd 2024-11-21T01:44:22,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/A/761482d657d8406abeb82a606c330b83 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/761482d657d8406abeb82a606c330b83 2024-11-21T01:44:22,488 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/761482d657d8406abeb82a606c330b83, entries=150, sequenceid=497, filesize=12.0 K 2024-11-21T01:44:22,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/B/4d08772d9cf641bba855d0af3c82f6b3 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/4d08772d9cf641bba855d0af3c82f6b3 2024-11-21T01:44:22,493 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/4d08772d9cf641bba855d0af3c82f6b3, entries=150, sequenceid=497, filesize=12.0 K 2024-11-21T01:44:22,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/.tmp/C/12b42db510ab4e9695bc691980f2c5fd as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/12b42db510ab4e9695bc691980f2c5fd 2024-11-21T01:44:22,497 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/12b42db510ab4e9695bc691980f2c5fd, entries=150, sequenceid=497, filesize=12.0 K 2024-11-21T01:44:22,498 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 06db9be3c4e6682aa8966442e1ef70cc in 859ms, sequenceid=497, compaction requested=true 2024-11-21T01:44:22,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 06db9be3c4e6682aa8966442e1ef70cc: 2024-11-21T01:44:22,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:22,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-21T01:44:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-21T01:44:22,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-21T01:44:22,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4110 sec 2024-11-21T01:44:22,502 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 2.4170 sec 2024-11-21T01:44:22,629 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T01:44:24,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T01:44:24,254 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:44:24,254 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T01:44:24,254 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-21T01:44:24,254 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-21T01:44:24,254 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-21T01:44:24,254 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-21T01:44:24,254 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-11-21T01:44:24,254 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T01:44:24,254 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1887 2024-11-21T01:44:24,254 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1856 2024-11-21T01:44:24,254 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T01:44:24,254 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 883 2024-11-21T01:44:24,254 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2645 rows 2024-11-21T01:44:24,254 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 875 2024-11-21T01:44:24,254 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2612 rows 2024-11-21T01:44:24,254 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T01:44:24,254 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T01:44:24,255 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:24,255 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:24,255 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:44:24,259 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-21T01:44:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T01:44:24,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-21T01:44:24,267 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732153464267"}]},"ts":"1732153464267"} 2024-11-21T01:44:24,269 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T01:44:24,269 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T01:44:24,270 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T01:44:24,275 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=06db9be3c4e6682aa8966442e1ef70cc, UNASSIGN}] 2024-11-21T01:44:24,276 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=06db9be3c4e6682aa8966442e1ef70cc, UNASSIGN 2024-11-21T01:44:24,278 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=06db9be3c4e6682aa8966442e1ef70cc, regionState=CLOSING, regionLocation=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:24,280 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=06db9be3c4e6682aa8966442e1ef70cc, UNASSIGN because future has completed 2024-11-21T01:44:24,280 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T01:44:24,281 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359}] 2024-11-21T01:44:24,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-21T01:44:24,437 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:44:24,438 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T01:44:24,438 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 06db9be3c4e6682aa8966442e1ef70cc, disabling compactions & flushes 2024-11-21T01:44:24,438 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:24,438 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:24,438 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. after waiting 0 ms 2024-11-21T01:44:24,438 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:24,439 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/ff226af3ec5b4b589df0a078dbef6065, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/5a3b6853766e43b7a8191715f0165620, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/3af4db8404554c95860f62541cc1d49a, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/c51a88b9ac724f8d8c19eb93c56d740f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/d75a6c22e60c4fe28e020c304570487e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/ac9eb303056a4f78836469d0a407b67f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/6826145630ad452897bc3501dd1b5734, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/1b791dec25c647bf9a1c6a92081aee9d, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/1218b22c39ae40e09c9b012016e562df, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/46d4a65094fa478bbcf31f7154438652, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/9113fe7a5b7847a88f4d4168fa455d39, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/58bdc8b25b494fc1acbfb5eede8f7455, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/a2201d51813448f6aa08feeda06a5564, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/ddb1c0d58b794e8e848eddf49afdf8e9, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/401069ea60404d94a5f9b749f2d11e04, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/3a003c18ebc64ddfb1b4480dd42259fd, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/06f3c9502dbb47eabe4d0f8d4600d747, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/1acb2875648244b6b469c12e837a962a, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/16c408688faf483993eb388de90d2c41, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/16a83b65d1ec4491a9f768418a6d74f1, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/3a94f45df01d4375b3bcf3b3f8b41a37, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/1b2c68dca45f414491ea78986d951b08, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/6d65d70a5c404824a61234047cd8db3f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/5482f3dd0e3a4171ad4f1be8463fd2d3, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/b1511f787562409a8caad24f68743c7c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/434cbcfe14f54dfc8b49116bd8e559d2, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/840c6794d4064519a1cec266925e6782, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/8fd0fe6486af45c9a2745dbc3437f796, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/8731da3435a748559103ab8eba382928, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/e3768d7a49574375a057af36b48c6baf, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/b38fec445b7f4583b2d58ac10fa991a3, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/3a2262f319514517863af260324ad5f5, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/993e53ddd8c145febc3fd27d1faf41b0, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/1b02005318bc44e38b22fec60fc4cdd7, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/5fe1308046e748c4b9a20b5279fee2d3] to archive 2024-11-21T01:44:24,442 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T01:44:24,447 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/ff226af3ec5b4b589df0a078dbef6065 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/ff226af3ec5b4b589df0a078dbef6065 2024-11-21T01:44:24,449 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/5a3b6853766e43b7a8191715f0165620 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/5a3b6853766e43b7a8191715f0165620 2024-11-21T01:44:24,450 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/3af4db8404554c95860f62541cc1d49a to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/3af4db8404554c95860f62541cc1d49a 2024-11-21T01:44:24,451 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/c51a88b9ac724f8d8c19eb93c56d740f to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/c51a88b9ac724f8d8c19eb93c56d740f 2024-11-21T01:44:24,452 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/d75a6c22e60c4fe28e020c304570487e to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/d75a6c22e60c4fe28e020c304570487e 2024-11-21T01:44:24,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/ac9eb303056a4f78836469d0a407b67f to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/ac9eb303056a4f78836469d0a407b67f 2024-11-21T01:44:24,455 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/6826145630ad452897bc3501dd1b5734 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/6826145630ad452897bc3501dd1b5734 2024-11-21T01:44:24,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/1b791dec25c647bf9a1c6a92081aee9d to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/1b791dec25c647bf9a1c6a92081aee9d 2024-11-21T01:44:24,458 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/1218b22c39ae40e09c9b012016e562df to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/1218b22c39ae40e09c9b012016e562df 2024-11-21T01:44:24,459 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/46d4a65094fa478bbcf31f7154438652 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/46d4a65094fa478bbcf31f7154438652 2024-11-21T01:44:24,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/9113fe7a5b7847a88f4d4168fa455d39 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/9113fe7a5b7847a88f4d4168fa455d39 2024-11-21T01:44:24,462 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/58bdc8b25b494fc1acbfb5eede8f7455 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/58bdc8b25b494fc1acbfb5eede8f7455 2024-11-21T01:44:24,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/a2201d51813448f6aa08feeda06a5564 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/a2201d51813448f6aa08feeda06a5564 2024-11-21T01:44:24,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/ddb1c0d58b794e8e848eddf49afdf8e9 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/ddb1c0d58b794e8e848eddf49afdf8e9 2024-11-21T01:44:24,466 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/401069ea60404d94a5f9b749f2d11e04 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/401069ea60404d94a5f9b749f2d11e04 2024-11-21T01:44:24,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/3a003c18ebc64ddfb1b4480dd42259fd to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/3a003c18ebc64ddfb1b4480dd42259fd 2024-11-21T01:44:24,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/06f3c9502dbb47eabe4d0f8d4600d747 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/06f3c9502dbb47eabe4d0f8d4600d747 2024-11-21T01:44:24,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/1acb2875648244b6b469c12e837a962a to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/1acb2875648244b6b469c12e837a962a 2024-11-21T01:44:24,471 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/16c408688faf483993eb388de90d2c41 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/16c408688faf483993eb388de90d2c41 2024-11-21T01:44:24,472 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/16a83b65d1ec4491a9f768418a6d74f1 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/16a83b65d1ec4491a9f768418a6d74f1 2024-11-21T01:44:24,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/3a94f45df01d4375b3bcf3b3f8b41a37 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/3a94f45df01d4375b3bcf3b3f8b41a37 2024-11-21T01:44:24,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/1b2c68dca45f414491ea78986d951b08 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/1b2c68dca45f414491ea78986d951b08 2024-11-21T01:44:24,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/6d65d70a5c404824a61234047cd8db3f to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/6d65d70a5c404824a61234047cd8db3f 2024-11-21T01:44:24,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/5482f3dd0e3a4171ad4f1be8463fd2d3 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/5482f3dd0e3a4171ad4f1be8463fd2d3 2024-11-21T01:44:24,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/b1511f787562409a8caad24f68743c7c to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/b1511f787562409a8caad24f68743c7c 2024-11-21T01:44:24,479 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/434cbcfe14f54dfc8b49116bd8e559d2 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/434cbcfe14f54dfc8b49116bd8e559d2 2024-11-21T01:44:24,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/840c6794d4064519a1cec266925e6782 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/840c6794d4064519a1cec266925e6782 2024-11-21T01:44:24,482 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/8fd0fe6486af45c9a2745dbc3437f796 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/8fd0fe6486af45c9a2745dbc3437f796 2024-11-21T01:44:24,483 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/8731da3435a748559103ab8eba382928 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/8731da3435a748559103ab8eba382928 2024-11-21T01:44:24,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/e3768d7a49574375a057af36b48c6baf to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/e3768d7a49574375a057af36b48c6baf 2024-11-21T01:44:24,486 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/b38fec445b7f4583b2d58ac10fa991a3 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/b38fec445b7f4583b2d58ac10fa991a3 2024-11-21T01:44:24,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/3a2262f319514517863af260324ad5f5 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/3a2262f319514517863af260324ad5f5 2024-11-21T01:44:24,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/993e53ddd8c145febc3fd27d1faf41b0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/993e53ddd8c145febc3fd27d1faf41b0 2024-11-21T01:44:24,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/1b02005318bc44e38b22fec60fc4cdd7 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/1b02005318bc44e38b22fec60fc4cdd7 2024-11-21T01:44:24,492 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/5fe1308046e748c4b9a20b5279fee2d3 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/5fe1308046e748c4b9a20b5279fee2d3 2024-11-21T01:44:24,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/aa2327f5f5244ebfa8b2844800ca58c6, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/2b53b601758c4af3902acee9e4c2f95a, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/00008e87abd04a3e912aa66a71c2bcf2, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/1543a422fe054b50895705843e33e1e0, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/bed7e243e8c64b459084fd404eeb9d97, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/86d5e7f297f74f06a7415eefe99e012e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/dc8fdde4342d4664b3a1c067add52dc9, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/d1ddd54ad24f411ea5ae6b3545a97104, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/16163cf5c76b4fe087e4ccce74b29d24, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/d1ca337fcb4240388cce930c3a1b8497, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/c12144a988df4d2f8c650573ab2d1827, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/f5cdfa4a1c7f47e8bc362b1e5ed53d18, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/d0fd7af99971427c87ac2a2e61ee5929, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/13bf9abbb76742da86e230efa77be623, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/8ca5e280b27542ceb7504b882835d0a5, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/df9e161c694548c1bb558ab2407ecae0, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/37a47146e6834d5daf3933276c587573, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/bd3258646b3f4ba59b41f6fc07fb21ec, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/54b918c104bf48a3948e22135422f52c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/8d01f5c435a0467eaca071b120d2fd0b, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/690a1385833348d393fbc9138b4d4d74, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/011b31c4c08a4e99a59d072f94273f4d, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/488dc2f1fd3946d98d761e75ff9d8049, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/bd29be2a9cb84883ac6cc12befa6d133, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/962472780dc849fe87796e9f98d29d40, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/628c53e431bb46098f8d64ac16d4c149, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/08e715696c544f468ee8057b7f67c9c4, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/754f3dce7b7d44dc8f73081f2bf9c7c0, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/e8a617c03f324a2283ff1ecb714c85c3, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/f8789e022b9645e693c571071ffb060d, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/f441de8ae21541aebcce5a3037a09867, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/560347b4a6a24f2ebb9c3c5a87191155, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/5cb6e55a6b72452ba0702bac00c0ad2b, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/e08bc71f6a984dfc997e7fd9e39ad5e0, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/63d8ca6089e04dd29880d6512c540c01] to archive 2024-11-21T01:44:24,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T01:44:24,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/aa2327f5f5244ebfa8b2844800ca58c6 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/aa2327f5f5244ebfa8b2844800ca58c6 2024-11-21T01:44:24,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/2b53b601758c4af3902acee9e4c2f95a to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/2b53b601758c4af3902acee9e4c2f95a 2024-11-21T01:44:24,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/00008e87abd04a3e912aa66a71c2bcf2 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/00008e87abd04a3e912aa66a71c2bcf2 2024-11-21T01:44:24,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/1543a422fe054b50895705843e33e1e0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/1543a422fe054b50895705843e33e1e0 2024-11-21T01:44:24,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/bed7e243e8c64b459084fd404eeb9d97 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/bed7e243e8c64b459084fd404eeb9d97 2024-11-21T01:44:24,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/86d5e7f297f74f06a7415eefe99e012e to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/86d5e7f297f74f06a7415eefe99e012e 2024-11-21T01:44:24,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/dc8fdde4342d4664b3a1c067add52dc9 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/dc8fdde4342d4664b3a1c067add52dc9 2024-11-21T01:44:24,519 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/d1ddd54ad24f411ea5ae6b3545a97104 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/d1ddd54ad24f411ea5ae6b3545a97104 2024-11-21T01:44:24,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/16163cf5c76b4fe087e4ccce74b29d24 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/16163cf5c76b4fe087e4ccce74b29d24 2024-11-21T01:44:24,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/d1ca337fcb4240388cce930c3a1b8497 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/d1ca337fcb4240388cce930c3a1b8497 2024-11-21T01:44:24,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/c12144a988df4d2f8c650573ab2d1827 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/c12144a988df4d2f8c650573ab2d1827 2024-11-21T01:44:24,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/f5cdfa4a1c7f47e8bc362b1e5ed53d18 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/f5cdfa4a1c7f47e8bc362b1e5ed53d18 2024-11-21T01:44:24,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/d0fd7af99971427c87ac2a2e61ee5929 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/d0fd7af99971427c87ac2a2e61ee5929 2024-11-21T01:44:24,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/13bf9abbb76742da86e230efa77be623 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/13bf9abbb76742da86e230efa77be623 2024-11-21T01:44:24,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/8ca5e280b27542ceb7504b882835d0a5 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/8ca5e280b27542ceb7504b882835d0a5 2024-11-21T01:44:24,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/df9e161c694548c1bb558ab2407ecae0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/df9e161c694548c1bb558ab2407ecae0 2024-11-21T01:44:24,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/37a47146e6834d5daf3933276c587573 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/37a47146e6834d5daf3933276c587573 2024-11-21T01:44:24,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/bd3258646b3f4ba59b41f6fc07fb21ec to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/bd3258646b3f4ba59b41f6fc07fb21ec 2024-11-21T01:44:24,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/54b918c104bf48a3948e22135422f52c to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/54b918c104bf48a3948e22135422f52c 2024-11-21T01:44:24,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/8d01f5c435a0467eaca071b120d2fd0b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/8d01f5c435a0467eaca071b120d2fd0b 2024-11-21T01:44:24,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/690a1385833348d393fbc9138b4d4d74 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/690a1385833348d393fbc9138b4d4d74 2024-11-21T01:44:24,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/011b31c4c08a4e99a59d072f94273f4d to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/011b31c4c08a4e99a59d072f94273f4d 2024-11-21T01:44:24,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/488dc2f1fd3946d98d761e75ff9d8049 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/488dc2f1fd3946d98d761e75ff9d8049 2024-11-21T01:44:24,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/bd29be2a9cb84883ac6cc12befa6d133 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/bd29be2a9cb84883ac6cc12befa6d133 2024-11-21T01:44:24,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/962472780dc849fe87796e9f98d29d40 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/962472780dc849fe87796e9f98d29d40 2024-11-21T01:44:24,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/628c53e431bb46098f8d64ac16d4c149 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/628c53e431bb46098f8d64ac16d4c149 2024-11-21T01:44:24,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/08e715696c544f468ee8057b7f67c9c4 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/08e715696c544f468ee8057b7f67c9c4 2024-11-21T01:44:24,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/754f3dce7b7d44dc8f73081f2bf9c7c0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/754f3dce7b7d44dc8f73081f2bf9c7c0 2024-11-21T01:44:24,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/e8a617c03f324a2283ff1ecb714c85c3 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/e8a617c03f324a2283ff1ecb714c85c3 2024-11-21T01:44:24,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/f8789e022b9645e693c571071ffb060d to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/f8789e022b9645e693c571071ffb060d 2024-11-21T01:44:24,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/f441de8ae21541aebcce5a3037a09867 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/f441de8ae21541aebcce5a3037a09867 2024-11-21T01:44:24,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/560347b4a6a24f2ebb9c3c5a87191155 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/560347b4a6a24f2ebb9c3c5a87191155 2024-11-21T01:44:24,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/5cb6e55a6b72452ba0702bac00c0ad2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/5cb6e55a6b72452ba0702bac00c0ad2b 2024-11-21T01:44:24,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/e08bc71f6a984dfc997e7fd9e39ad5e0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/e08bc71f6a984dfc997e7fd9e39ad5e0 2024-11-21T01:44:24,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/63d8ca6089e04dd29880d6512c540c01 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/63d8ca6089e04dd29880d6512c540c01 2024-11-21T01:44:24,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/46b41407933d476cbd1c277cbaec8bf6, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/9beea32024024c2098d385cd39b9cf25, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/bf8b284189a646708c11bc09d9dae7aa, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/7e4454cd9ee94e40a0d4d5e1db39013d, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/1884ee141c894eabbf9fb970faf3e3b8, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/9f62f9329adb443b8bd1e10f46cd7814, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/1e5856b9f02b4ff0bd7e1e6b2de02f27, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/0cd4cc1dffdc474ca5da18b13ed046be, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/05911bffc59c4285916009efda0b6c20, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/a7e775d529484c3ba8258e29d309ea77, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/51397f5a15dc457886c2183bc7260809, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/221c9f218b0b4951aaa041fa2ded3e7e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/856c3386b59e4a5b81d5739d34631049, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/70e78e3c4ce346aaa0f6f06f0b9f7fc0, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/c7822297793e4df19442051e4beae45f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/98260bc22c7c4257b6e93bbc0e85a2df, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/cb03140dc82c4e5cbf1f2178fc7ad468, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/66e5280aad65408dbd09414e6bcd093f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/001bd063bf1040468625035b1a337510, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/7179161c07744e90b2904e16699401d0, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/9f3be1919361425cbc4eca2266e2cb86, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/91d9c5c901d543b8a01f016980301597, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/c61532d33e3645f3a00d332d62f4bdfe, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/dac486fa61524c7ca7b0248a78eb22e0, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/69544680ac6f4ac7ad1a46b144c64418, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/13ecd9b1f56c4f96879b19604f10bf00, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/228374ba78e04dc58afbc52e3ce32b68, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/27e5ad8dee604086a871ee6e28138d6a, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/67c7227340c640498295bff4ed7cf364, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/8cb3543717f74efba2b195487b349285, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/7af288d1e7f944f68f9ea4c6e0527ba7, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/8cc8e2b55df44e768589e8d15edec034, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/0303c49b8a40456a81764d0cb7eb42c9, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/4be627756b614fe196897c1501a094bc, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/7a4c990255394730b965c649646563c7] to archive 2024-11-21T01:44:24,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T01:44:24,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/46b41407933d476cbd1c277cbaec8bf6 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/46b41407933d476cbd1c277cbaec8bf6 2024-11-21T01:44:24,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/9beea32024024c2098d385cd39b9cf25 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/9beea32024024c2098d385cd39b9cf25 2024-11-21T01:44:24,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/bf8b284189a646708c11bc09d9dae7aa to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/bf8b284189a646708c11bc09d9dae7aa 2024-11-21T01:44:24,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/7e4454cd9ee94e40a0d4d5e1db39013d to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/7e4454cd9ee94e40a0d4d5e1db39013d 2024-11-21T01:44:24,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/1884ee141c894eabbf9fb970faf3e3b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/1884ee141c894eabbf9fb970faf3e3b8 2024-11-21T01:44:24,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/9f62f9329adb443b8bd1e10f46cd7814 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/9f62f9329adb443b8bd1e10f46cd7814 2024-11-21T01:44:24,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/1e5856b9f02b4ff0bd7e1e6b2de02f27 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/1e5856b9f02b4ff0bd7e1e6b2de02f27 2024-11-21T01:44:24,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/0cd4cc1dffdc474ca5da18b13ed046be to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/0cd4cc1dffdc474ca5da18b13ed046be 2024-11-21T01:44:24,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/05911bffc59c4285916009efda0b6c20 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/05911bffc59c4285916009efda0b6c20 2024-11-21T01:44:24,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/a7e775d529484c3ba8258e29d309ea77 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/a7e775d529484c3ba8258e29d309ea77 2024-11-21T01:44:24,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/51397f5a15dc457886c2183bc7260809 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/51397f5a15dc457886c2183bc7260809 2024-11-21T01:44:24,579 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/221c9f218b0b4951aaa041fa2ded3e7e to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/221c9f218b0b4951aaa041fa2ded3e7e 2024-11-21T01:44:24,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/856c3386b59e4a5b81d5739d34631049 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/856c3386b59e4a5b81d5739d34631049 2024-11-21T01:44:24,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/70e78e3c4ce346aaa0f6f06f0b9f7fc0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/70e78e3c4ce346aaa0f6f06f0b9f7fc0 2024-11-21T01:44:24,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/c7822297793e4df19442051e4beae45f to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/c7822297793e4df19442051e4beae45f 2024-11-21T01:44:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-21T01:44:24,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/98260bc22c7c4257b6e93bbc0e85a2df to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/98260bc22c7c4257b6e93bbc0e85a2df 2024-11-21T01:44:24,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/cb03140dc82c4e5cbf1f2178fc7ad468 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/cb03140dc82c4e5cbf1f2178fc7ad468 2024-11-21T01:44:24,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/66e5280aad65408dbd09414e6bcd093f to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/66e5280aad65408dbd09414e6bcd093f 2024-11-21T01:44:24,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/001bd063bf1040468625035b1a337510 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/001bd063bf1040468625035b1a337510 2024-11-21T01:44:24,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/7179161c07744e90b2904e16699401d0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/7179161c07744e90b2904e16699401d0 2024-11-21T01:44:24,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/9f3be1919361425cbc4eca2266e2cb86 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/9f3be1919361425cbc4eca2266e2cb86 2024-11-21T01:44:24,596 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/91d9c5c901d543b8a01f016980301597 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/91d9c5c901d543b8a01f016980301597 2024-11-21T01:44:24,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/c61532d33e3645f3a00d332d62f4bdfe to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/c61532d33e3645f3a00d332d62f4bdfe 2024-11-21T01:44:24,599 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/dac486fa61524c7ca7b0248a78eb22e0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/dac486fa61524c7ca7b0248a78eb22e0 2024-11-21T01:44:24,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/69544680ac6f4ac7ad1a46b144c64418 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/69544680ac6f4ac7ad1a46b144c64418 2024-11-21T01:44:24,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/13ecd9b1f56c4f96879b19604f10bf00 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/13ecd9b1f56c4f96879b19604f10bf00 2024-11-21T01:44:24,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/228374ba78e04dc58afbc52e3ce32b68 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/228374ba78e04dc58afbc52e3ce32b68 2024-11-21T01:44:24,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/27e5ad8dee604086a871ee6e28138d6a to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/27e5ad8dee604086a871ee6e28138d6a 2024-11-21T01:44:24,605 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/67c7227340c640498295bff4ed7cf364 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/67c7227340c640498295bff4ed7cf364 2024-11-21T01:44:24,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/8cb3543717f74efba2b195487b349285 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/8cb3543717f74efba2b195487b349285 2024-11-21T01:44:24,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/7af288d1e7f944f68f9ea4c6e0527ba7 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/7af288d1e7f944f68f9ea4c6e0527ba7 2024-11-21T01:44:24,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/8cc8e2b55df44e768589e8d15edec034 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/8cc8e2b55df44e768589e8d15edec034 2024-11-21T01:44:24,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/0303c49b8a40456a81764d0cb7eb42c9 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/0303c49b8a40456a81764d0cb7eb42c9 2024-11-21T01:44:24,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/4be627756b614fe196897c1501a094bc to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/4be627756b614fe196897c1501a094bc 2024-11-21T01:44:24,612 DEBUG [StoreCloser-TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/7a4c990255394730b965c649646563c7 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/7a4c990255394730b965c649646563c7 2024-11-21T01:44:24,617 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/recovered.edits/500.seqid, newMaxSeqId=500, maxSeqId=1 2024-11-21T01:44:24,618 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc. 2024-11-21T01:44:24,618 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 06db9be3c4e6682aa8966442e1ef70cc: Waiting for close lock at 1732153464438Running coprocessor pre-close hooks at 1732153464438Disabling compacts and flushes for region at 1732153464438Disabling writes for close at 1732153464438Writing region close event to WAL at 1732153464613 (+175 ms)Running coprocessor post-close hooks at 1732153464617 (+4 ms)Closed at 1732153464618 (+1 ms) 2024-11-21T01:44:24,620 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:44:24,621 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=06db9be3c4e6682aa8966442e1ef70cc, regionState=CLOSED 2024-11-21T01:44:24,623 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 because future has completed 2024-11-21T01:44:24,625 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-21T01:44:24,626 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 06db9be3c4e6682aa8966442e1ef70cc, server=b93ada835ef8,41129,1732153434359 in 344 msec 2024-11-21T01:44:24,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-21T01:44:24,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=06db9be3c4e6682aa8966442e1ef70cc, UNASSIGN in 351 msec 2024-11-21T01:44:24,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-21T01:44:24,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 358 msec 2024-11-21T01:44:24,631 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732153464631"}]},"ts":"1732153464631"} 2024-11-21T01:44:24,632 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T01:44:24,632 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T01:44:24,634 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 372 msec 2024-11-21T01:44:24,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-21T01:44:24,893 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:44:24,896 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-21T01:44:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:44:24,902 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:44:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-21T01:44:24,903 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:44:24,909 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:44:24,913 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A, FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B, FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C, FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/recovered.edits] 2024-11-21T01:44:24,916 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/761482d657d8406abeb82a606c330b83 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/761482d657d8406abeb82a606c330b83 2024-11-21T01:44:24,917 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/dee0875a793d4d6cb79d7cea1e8ef240 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/dee0875a793d4d6cb79d7cea1e8ef240 2024-11-21T01:44:24,919 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/fe90cc7e11874d0f8c882b8749f0a9f4 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/A/fe90cc7e11874d0f8c882b8749f0a9f4 2024-11-21T01:44:24,921 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/0608828926634165a01916cb7a0d5782 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/0608828926634165a01916cb7a0d5782 2024-11-21T01:44:24,923 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/4d08772d9cf641bba855d0af3c82f6b3 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/4d08772d9cf641bba855d0af3c82f6b3 2024-11-21T01:44:24,924 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/97eac630e72d476580de677af281ca56 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/B/97eac630e72d476580de677af281ca56 2024-11-21T01:44:24,927 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/12b42db510ab4e9695bc691980f2c5fd to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/12b42db510ab4e9695bc691980f2c5fd 2024-11-21T01:44:24,928 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/75d768714fb6429cacda071a8a120af8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/75d768714fb6429cacda071a8a120af8 2024-11-21T01:44:24,930 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/779417a26c5045dabfc44a8b6ace6691 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/C/779417a26c5045dabfc44a8b6ace6691 2024-11-21T01:44:24,933 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/recovered.edits/500.seqid to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc/recovered.edits/500.seqid 2024-11-21T01:44:24,934 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/06db9be3c4e6682aa8966442e1ef70cc 2024-11-21T01:44:24,934 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T01:44:24,939 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:44:24,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-21T01:44:24,947 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T01:44:24,979 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T01:44:24,980 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:44:24,980 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T01:44:24,980 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732153464980"}]},"ts":"9223372036854775807"} 2024-11-21T01:44:24,986 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T01:44:24,986 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 06db9be3c4e6682aa8966442e1ef70cc, NAME => 'TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T01:44:24,986 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T01:44:24,986 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732153464986"}]},"ts":"9223372036854775807"} 2024-11-21T01:44:24,989 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T01:44:24,990 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:44:24,991 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 93 msec 2024-11-21T01:44:25,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-21T01:44:25,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T01:44:25,014 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:44:25,027 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1740360738_22 at /127.0.0.1:42430 [Waiting for operation #68] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1740360738_22 at /127.0.0.1:55520 [Waiting for operation #315] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;b93ada835ef8:41129-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1139353644_22 at /127.0.0.1:58680 [Waiting for operation #322] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=608 (was 481) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2901 (was 262) - AvailableMemoryMB LEAK? - 2024-11-21T01:44:25,037 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=608, ProcessCount=11, AvailableMemoryMB=2900 2024-11-21T01:44:25,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T01:44:25,038 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T01:44:25,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T01:44:25,040 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T01:44:25,040 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:25,040 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-21T01:44:25,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-21T01:44:25,041 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T01:44:25,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741951_1127 (size=1061) 2024-11-21T01:44:25,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-21T01:44:25,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-21T01:44:25,449 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d 2024-11-21T01:44:25,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741952_1128 (size=53) 2024-11-21T01:44:25,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-21T01:44:25,862 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:44:25,862 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5835087c277e0e7bdbee836efeecc2f0, disabling compactions & flushes 2024-11-21T01:44:25,862 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:25,862 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:25,862 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. after waiting 0 ms 2024-11-21T01:44:25,863 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:25,863 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:25,863 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5835087c277e0e7bdbee836efeecc2f0: Waiting for close lock at 1732153465862Disabling compacts and flushes for region at 1732153465862Disabling writes for close at 1732153465863 (+1 ms)Writing region close event to WAL at 1732153465863Closed at 1732153465863 2024-11-21T01:44:25,864 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T01:44:25,864 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732153465864"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732153465864"}]},"ts":"1732153465864"} 2024-11-21T01:44:25,866 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T01:44:25,867 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T01:44:25,868 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732153465867"}]},"ts":"1732153465867"} 2024-11-21T01:44:25,869 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T01:44:25,869 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5835087c277e0e7bdbee836efeecc2f0, ASSIGN}] 2024-11-21T01:44:25,871 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5835087c277e0e7bdbee836efeecc2f0, ASSIGN 2024-11-21T01:44:25,871 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5835087c277e0e7bdbee836efeecc2f0, ASSIGN; state=OFFLINE, location=b93ada835ef8,41129,1732153434359; forceNewPlan=false, retain=false 2024-11-21T01:44:26,022 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=5835087c277e0e7bdbee836efeecc2f0, regionState=OPENING, regionLocation=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:26,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5835087c277e0e7bdbee836efeecc2f0, ASSIGN because future has completed 2024-11-21T01:44:26,025 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359}] 2024-11-21T01:44:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-21T01:44:26,181 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:26,181 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} 2024-11-21T01:44:26,182 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:26,182 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:44:26,182 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:26,182 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:26,184 INFO [StoreOpener-5835087c277e0e7bdbee836efeecc2f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:26,185 INFO [StoreOpener-5835087c277e0e7bdbee836efeecc2f0-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T01:44:26,186 INFO [StoreOpener-5835087c277e0e7bdbee836efeecc2f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5835087c277e0e7bdbee836efeecc2f0 columnFamilyName A 2024-11-21T01:44:26,187 DEBUG [StoreOpener-5835087c277e0e7bdbee836efeecc2f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:26,187 INFO [StoreOpener-5835087c277e0e7bdbee836efeecc2f0-1 {}] regionserver.HStore(327): Store=5835087c277e0e7bdbee836efeecc2f0/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:44:26,187 INFO [StoreOpener-5835087c277e0e7bdbee836efeecc2f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:26,188 INFO [StoreOpener-5835087c277e0e7bdbee836efeecc2f0-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T01:44:26,189 INFO [StoreOpener-5835087c277e0e7bdbee836efeecc2f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5835087c277e0e7bdbee836efeecc2f0 columnFamilyName B 2024-11-21T01:44:26,189 DEBUG [StoreOpener-5835087c277e0e7bdbee836efeecc2f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:26,189 INFO [StoreOpener-5835087c277e0e7bdbee836efeecc2f0-1 {}] regionserver.HStore(327): Store=5835087c277e0e7bdbee836efeecc2f0/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:44:26,189 INFO [StoreOpener-5835087c277e0e7bdbee836efeecc2f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:26,190 INFO [StoreOpener-5835087c277e0e7bdbee836efeecc2f0-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T01:44:26,191 INFO [StoreOpener-5835087c277e0e7bdbee836efeecc2f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5835087c277e0e7bdbee836efeecc2f0 columnFamilyName C 2024-11-21T01:44:26,191 DEBUG [StoreOpener-5835087c277e0e7bdbee836efeecc2f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:26,191 INFO [StoreOpener-5835087c277e0e7bdbee836efeecc2f0-1 {}] regionserver.HStore(327): Store=5835087c277e0e7bdbee836efeecc2f0/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:44:26,191 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:26,191 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:26,192 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:26,193 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:26,194 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:26,194 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:26,194 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T01:44:26,195 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:26,197 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T01:44:26,198 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 5835087c277e0e7bdbee836efeecc2f0; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61947576, jitterRate=-0.07690918445587158}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T01:44:26,198 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:26,199 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 5835087c277e0e7bdbee836efeecc2f0: Running coprocessor pre-open hook at 1732153466182Writing region info on filesystem at 1732153466182Initializing all the Stores at 1732153466183 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732153466183Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732153466184 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732153466184Cleaning up temporary data from old regions at 1732153466194 (+10 ms)Running coprocessor post-open hooks at 1732153466198 (+4 ms)Region opened successfully at 1732153466199 (+1 ms) 2024-11-21T01:44:26,199 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., pid=34, masterSystemTime=1732153466177 2024-11-21T01:44:26,201 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:26,201 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:26,202 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=5835087c277e0e7bdbee836efeecc2f0, regionState=OPEN, openSeqNum=2, regionLocation=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:26,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 because future has completed 2024-11-21T01:44:26,209 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-21T01:44:26,209 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 in 183 msec 2024-11-21T01:44:26,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-21T01:44:26,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5835087c277e0e7bdbee836efeecc2f0, ASSIGN in 341 msec 2024-11-21T01:44:26,212 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T01:44:26,212 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732153466212"}]},"ts":"1732153466212"} 2024-11-21T01:44:26,214 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T01:44:26,215 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T01:44:26,216 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1770 sec 2024-11-21T01:44:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-21T01:44:27,193 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:44:27,194 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a76d180, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:27,194 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:44:27,194 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:44:27,196 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:44:27,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:44:27,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:44:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ef7550, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:44:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:44:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:27,198 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40164, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:44:27,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d5b06de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:27,199 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:44:27,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:44:27,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:44:27,202 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:44:27,203 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T01:44:27,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is b93ada835ef8,36607,1732153433607 2024-11-21T01:44:27,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1b7a241d 2024-11-21T01:44:27,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T01:44:27,205 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40174, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T01:44:27,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T01:44:27,210 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T01:44:27,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-21T01:44:27,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741953_1129 (size=1097) 2024-11-21T01:44:27,631 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-21T01:44:27,631 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-21T01:44:27,636 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T01:44:27,643 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5835087c277e0e7bdbee836efeecc2f0, REOPEN/MOVE}] 2024-11-21T01:44:27,644 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5835087c277e0e7bdbee836efeecc2f0, REOPEN/MOVE 2024-11-21T01:44:27,645 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=5835087c277e0e7bdbee836efeecc2f0, regionState=CLOSING, regionLocation=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:27,646 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5835087c277e0e7bdbee836efeecc2f0, REOPEN/MOVE because future has completed 2024-11-21T01:44:27,647 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T01:44:27,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359}] 2024-11-21T01:44:27,799 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:27,800 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T01:44:27,800 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 5835087c277e0e7bdbee836efeecc2f0, disabling compactions & flushes 2024-11-21T01:44:27,800 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:27,800 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:27,800 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. after waiting 0 ms 2024-11-21T01:44:27,800 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:27,804 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T01:44:27,805 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:27,805 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 5835087c277e0e7bdbee836efeecc2f0: Waiting for close lock at 1732153467800Running coprocessor pre-close hooks at 1732153467800Disabling compacts and flushes for region at 1732153467800Disabling writes for close at 1732153467800Writing region close event to WAL at 1732153467800Running coprocessor post-close hooks at 1732153467805 (+5 ms)Closed at 1732153467805 2024-11-21T01:44:27,805 WARN [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: 5835087c277e0e7bdbee836efeecc2f0 to self. 2024-11-21T01:44:27,807 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:27,808 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=5835087c277e0e7bdbee836efeecc2f0, regionState=CLOSED 2024-11-21T01:44:27,810 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 because future has completed 2024-11-21T01:44:27,812 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-21T01:44:27,812 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure 5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 in 164 msec 2024-11-21T01:44:27,813 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5835087c277e0e7bdbee836efeecc2f0, REOPEN/MOVE; state=CLOSED, location=b93ada835ef8,41129,1732153434359; forceNewPlan=false, retain=true 2024-11-21T01:44:27,964 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=5835087c277e0e7bdbee836efeecc2f0, regionState=OPENING, regionLocation=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:27,967 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5835087c277e0e7bdbee836efeecc2f0, REOPEN/MOVE because future has completed 2024-11-21T01:44:27,967 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359}] 2024-11-21T01:44:28,123 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:28,123 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} 2024-11-21T01:44:28,124 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:28,124 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:44:28,124 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:28,124 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:28,126 INFO [StoreOpener-5835087c277e0e7bdbee836efeecc2f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:28,127 INFO [StoreOpener-5835087c277e0e7bdbee836efeecc2f0-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T01:44:28,133 INFO [StoreOpener-5835087c277e0e7bdbee836efeecc2f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5835087c277e0e7bdbee836efeecc2f0 columnFamilyName A 2024-11-21T01:44:28,135 DEBUG [StoreOpener-5835087c277e0e7bdbee836efeecc2f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:28,136 INFO [StoreOpener-5835087c277e0e7bdbee836efeecc2f0-1 {}] regionserver.HStore(327): Store=5835087c277e0e7bdbee836efeecc2f0/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:44:28,136 INFO [StoreOpener-5835087c277e0e7bdbee836efeecc2f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:28,137 INFO [StoreOpener-5835087c277e0e7bdbee836efeecc2f0-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T01:44:28,137 INFO [StoreOpener-5835087c277e0e7bdbee836efeecc2f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5835087c277e0e7bdbee836efeecc2f0 columnFamilyName B 2024-11-21T01:44:28,137 DEBUG [StoreOpener-5835087c277e0e7bdbee836efeecc2f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:28,138 INFO [StoreOpener-5835087c277e0e7bdbee836efeecc2f0-1 {}] regionserver.HStore(327): Store=5835087c277e0e7bdbee836efeecc2f0/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:44:28,138 INFO [StoreOpener-5835087c277e0e7bdbee836efeecc2f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:28,138 INFO [StoreOpener-5835087c277e0e7bdbee836efeecc2f0-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T01:44:28,139 INFO [StoreOpener-5835087c277e0e7bdbee836efeecc2f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5835087c277e0e7bdbee836efeecc2f0 columnFamilyName C 2024-11-21T01:44:28,139 DEBUG [StoreOpener-5835087c277e0e7bdbee836efeecc2f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:28,139 INFO [StoreOpener-5835087c277e0e7bdbee836efeecc2f0-1 {}] regionserver.HStore(327): Store=5835087c277e0e7bdbee836efeecc2f0/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:44:28,139 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:28,139 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:28,140 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:28,141 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:28,142 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:28,142 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:28,143 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T01:44:28,144 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:28,145 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 5835087c277e0e7bdbee836efeecc2f0; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71533237, jitterRate=0.06592829525470734}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T01:44:28,145 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:28,146 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 5835087c277e0e7bdbee836efeecc2f0: Running coprocessor pre-open hook at 1732153468124Writing region info on filesystem at 1732153468124Initializing all the Stores at 1732153468125 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732153468125Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732153468125Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732153468125Cleaning up temporary data from old regions at 1732153468142 (+17 ms)Running coprocessor post-open hooks at 1732153468145 (+3 ms)Region opened successfully at 1732153468146 (+1 ms) 2024-11-21T01:44:28,147 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., pid=39, masterSystemTime=1732153468119 2024-11-21T01:44:28,150 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:28,150 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:28,150 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=5835087c277e0e7bdbee836efeecc2f0, regionState=OPEN, openSeqNum=5, regionLocation=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:28,152 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 because future has completed 2024-11-21T01:44:28,155 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-21T01:44:28,155 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 in 186 msec 2024-11-21T01:44:28,157 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-21T01:44:28,157 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5835087c277e0e7bdbee836efeecc2f0, REOPEN/MOVE in 512 msec 2024-11-21T01:44:28,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-21T01:44:28,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 524 msec 2024-11-21T01:44:28,165 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 951 msec 2024-11-21T01:44:28,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-21T01:44:28,167 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:44:28,168 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c3fe3db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:28,168 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:44:28,168 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:44:28,170 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:44:28,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:44:28,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:44:28,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49578a42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:28,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:44:28,171 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:44:28,171 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:28,172 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:44:28,172 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65fc31d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:28,173 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f9c3dcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:28,173 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:44:28,173 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:44:28,174 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:44:28,174 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:44:28,174 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:44:28,174 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3177e8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:28,174 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:44:28,174 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:44:28,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:28,175 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40210, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:44:28,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5500ff2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:28,177 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dde8260, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:28,177 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:44:28,177 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:44:28,178 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:44:28,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:44:28,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:44:28,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a2bf826, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:28,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:44:28,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:44:28,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:28,179 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:44:28,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45eac05b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:28,181 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ba7f2f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:28,181 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:44:28,181 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:44:28,181 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:44:28,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:44:28,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:44:28,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@362aacdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:28,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:44:28,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:44:28,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:28,183 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40236, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:44:28,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7093d32b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:28,184 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ea1a430, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:28,184 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:44:28,184 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:44:28,185 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:44:28,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:44:28,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:44:28,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750733f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:28,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:44:28,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:44:28,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:28,186 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40256, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:44:28,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1748e009, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:28,187 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@598c3817, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:28,187 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:44:28,187 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:44:28,188 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:44:28,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:44:28,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:44:28,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28bec98f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:28,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:44:28,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:44:28,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:28,189 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:44:28,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@303589b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:28,190 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@292bddff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:28,190 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:44:28,191 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:44:28,191 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:44:28,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:44:28,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:44:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@778c9c73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:44:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:44:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:28,193 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40290, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:44:28,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a0d289b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:28,194 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f60d5da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:28,194 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:44:28,194 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:44:28,195 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:44:28,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:44:28,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:44:28,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2d70d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:28,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:44:28,196 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:44:28,196 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:28,196 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40312, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:44:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7461d5cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:28,198 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@139f617f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:28,198 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:44:28,198 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:44:28,198 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:44:28,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:44:28,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:44:28,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3675059a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:28,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:44:28,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:44:28,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:28,200 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:44:28,201 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48a91d9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:28,201 DEBUG [Thread-657 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:44:28,201 DEBUG [Thread-648 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:44:28,201 DEBUG [Thread-654 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:44:28,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:44:28,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:44:28,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:44:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:44:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:44:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:44:28,203 DEBUG [Thread-663 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:44:28,204 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42316, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:44:28,204 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42326, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:44:28,204 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42302, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:44:28,204 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:44:28,205 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:44:28,205 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42328, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:44:28,206 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:44:28,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5] 2024-11-21T01:44:28,209 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5] 2024-11-21T01:44:28,210 DEBUG [Thread-646 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:44:28,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5] 2024-11-21T01:44:28,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-21T01:44:28,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5] 2024-11-21T01:44:28,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T01:44:28,212 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:44:28,212 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:44:28,212 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:44:28,213 DEBUG [Thread-652 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:44:28,213 DEBUG [Thread-661 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:44:28,213 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:44:28,213 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42340, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:44:28,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:44:28,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:44:28,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:44:28,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:44:28,214 DEBUG [Thread-650 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:44:28,214 DEBUG [Thread-659 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:44:28,215 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42352, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:44:28,216 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:44:28,216 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42364, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:44:28,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5] 2024-11-21T01:44:28,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5] 2024-11-21T01:44:28,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5] 2024-11-21T01:44:28,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:44:28,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:44:28,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:44:28,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:44:28,223 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42372, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:44:28,223 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42376, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:44:28,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5] 2024-11-21T01:44:28,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5] 2024-11-21T01:44:28,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:28,239 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5835087c277e0e7bdbee836efeecc2f0 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-21T01:44:28,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=A 2024-11-21T01:44:28,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:28,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=B 2024-11-21T01:44:28,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:28,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=C 2024-11-21T01:44:28,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T01:44:28,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d684a9ae513f4f85bc96256a900f807b_5835087c277e0e7bdbee836efeecc2f0 is 50, key is test_row_0/A:col10/1732153468234/Put/seqid=0 2024-11-21T01:44:28,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732153468275, max timestamp=1732153468317Num uniques 859; 2024-11-21T01:44:28,336 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T01:44:28,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=191, cellSize=8746, totalHeapSize=20104, min timestamp=1732153468275, max timestamp=1732153468317Num uniques 859; 2024-11-21T01:44:28,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=8746, suffix heap size=117064,new segment heap size=20104 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=191 2024-11-21T01:44:28,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732153468275, max timestamp=1732153468317Num uniques 859; 2024-11-21T01:44:28,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T01:44:28,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T01:44:28,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:28,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:28,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:28,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:28,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:28,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741954_1130 (size=21918) 2024-11-21T01:44:28,375 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:28,380 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d684a9ae513f4f85bc96256a900f807b_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d684a9ae513f4f85bc96256a900f807b_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:28,381 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/a8fdcb23f6fa4860b229720a998831d1, store: [table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:28,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/a8fdcb23f6fa4860b229720a998831d1 is 175, key is test_row_0/A:col10/1732153468234/Put/seqid=0 2024-11-21T01:44:28,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153528404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:28,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:28,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153528414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:28,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:28,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:28,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:28,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153528415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:28,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:28,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:28,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:28,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:28,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153528418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:28,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:28,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:28,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:28,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:28,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42376 deadline: 1732153528420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:28,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741955_1131 (size=65323) 2024-11-21T01:44:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T01:44:28,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T01:44:28,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:28,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:28,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:28,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:28,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:28,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:28,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153528533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:28,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:28,543 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:28,543 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:28,543 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:28,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153528537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:28,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:28,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153528537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:28,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153528543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:28,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:28,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:28,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:28,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42376 deadline: 1732153528543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:28,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:28,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:28,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:28,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:28,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:28,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:28,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:28,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:28,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:28,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T01:44:28,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:28,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:28,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:28,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:28,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:28,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153528754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:28,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:28,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:28,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:28,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153528754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:28,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:28,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:28,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:28,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42376 deadline: 1732153528754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:28,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:28,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:28,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:28,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153528757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:28,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153528754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:28,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:28,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:28,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:28,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:28,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:28,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T01:44:28,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T01:44:28,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:28,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:28,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:28,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:28,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:28,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:28,851 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/a8fdcb23f6fa4860b229720a998831d1 2024-11-21T01:44:28,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/70e8711ecfaf4b7992f91cfce9e4743f is 50, key is test_row_0/B:col10/1732153468234/Put/seqid=0 2024-11-21T01:44:28,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741956_1132 (size=12001) 2024-11-21T01:44:28,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/70e8711ecfaf4b7992f91cfce9e4743f 2024-11-21T01:44:28,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T01:44:28,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:28,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:28,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:28,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:28,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:28,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:29,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/4f3f4859f83142af94aab4c72c128fd0 is 50, key is test_row_0/C:col10/1732153468234/Put/seqid=0 2024-11-21T01:44:29,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741957_1133 (size=12001) 2024-11-21T01:44:29,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153529063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:29,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:29,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153529063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:29,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153529073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:29,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:29,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42376 deadline: 1732153529073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:29,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:29,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:29,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153529074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:29,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:29,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T01:44:29,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:29,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:29,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:29,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:29,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:29,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T01:44:29,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:29,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:29,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:29,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:29,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T01:44:29,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/4f3f4859f83142af94aab4c72c128fd0 2024-11-21T01:44:29,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/a8fdcb23f6fa4860b229720a998831d1 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/a8fdcb23f6fa4860b229720a998831d1 2024-11-21T01:44:29,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/a8fdcb23f6fa4860b229720a998831d1, entries=350, sequenceid=15, filesize=63.8 K 2024-11-21T01:44:29,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/70e8711ecfaf4b7992f91cfce9e4743f as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/70e8711ecfaf4b7992f91cfce9e4743f 2024-11-21T01:44:29,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T01:44:29,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:29,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:29,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:29,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:29,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:29,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/70e8711ecfaf4b7992f91cfce9e4743f, entries=150, sequenceid=15, filesize=11.7 K 2024-11-21T01:44:29,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/4f3f4859f83142af94aab4c72c128fd0 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/4f3f4859f83142af94aab4c72c128fd0 2024-11-21T01:44:29,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/4f3f4859f83142af94aab4c72c128fd0, entries=150, sequenceid=15, filesize=11.7 K 2024-11-21T01:44:29,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=128.91 KB/132002 for 5835087c277e0e7bdbee836efeecc2f0 in 1254ms, sequenceid=15, compaction requested=false 2024-11-21T01:44:29,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:29,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:29,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5835087c277e0e7bdbee836efeecc2f0 3/3 column families, dataSize=135.62 KB heapSize=388.23 KB 2024-11-21T01:44:29,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=A 2024-11-21T01:44:29,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:29,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=B 2024-11-21T01:44:29,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:29,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=C 2024-11-21T01:44:29,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:29,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T01:44:29,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:29,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:29,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:29,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:29,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:29,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112175727f2334234b64af1e7d4a5dc194fe_5835087c277e0e7bdbee836efeecc2f0 is 50, key is test_row_0/A:col10/1732153468415/Put/seqid=0 2024-11-21T01:44:29,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:29,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42376 deadline: 1732153529611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:29,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:29,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153529612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:29,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:29,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153529616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:29,633 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:29,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:29,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153529623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:29,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:29,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153529624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:29,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741958_1134 (size=14594) 2024-11-21T01:44:29,686 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:29,701 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112175727f2334234b64af1e7d4a5dc194fe_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112175727f2334234b64af1e7d4a5dc194fe_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:29,702 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/d4bac36c7bb7468f990eb3a9296a6b9c, store: [table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:29,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/d4bac36c7bb7468f990eb3a9296a6b9c is 175, key is test_row_0/A:col10/1732153468415/Put/seqid=0 2024-11-21T01:44:29,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741959_1135 (size=39549) 2024-11-21T01:44:29,714 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=46.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/d4bac36c7bb7468f990eb3a9296a6b9c 2024-11-21T01:44:29,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/3eae683dcae342568f5bee745339d7e1 is 50, key is test_row_0/B:col10/1732153468415/Put/seqid=0 2024-11-21T01:44:29,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:29,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42376 deadline: 1732153529733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:29,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:29,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741960_1136 (size=12001) 2024-11-21T01:44:29,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:29,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153529743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:29,749 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,749 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,749 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:29,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:29,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153529743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:29,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:29,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:29,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153529743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:29,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:29,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:29,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153529743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:29,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:29,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T01:44:29,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:29,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:29,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:29,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:29,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:29,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:29,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T01:44:29,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:29,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:29,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:29,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:29,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:29,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:29,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:29,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42376 deadline: 1732153529943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:29,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:29,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:29,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153529953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:29,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:29,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:29,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153529963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:29,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:29,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153529963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:29,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:29,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:29,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153529964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:29,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:29,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:29,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:30,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T01:44:30,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:30,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:30,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:30,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:30,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:30,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:30,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/3eae683dcae342568f5bee745339d7e1 2024-11-21T01:44:30,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/b53592f8387e46bb86b238df551b268b is 50, key is test_row_0/C:col10/1732153468415/Put/seqid=0 2024-11-21T01:44:30,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741961_1137 (size=12001) 2024-11-21T01:44:30,179 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T01:44:30,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T01:44:30,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:30,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:30,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:30,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:30,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:30,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42376 deadline: 1732153530258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:30,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153530263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:30,268 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:30,268 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:30,268 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:30,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:30,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153530273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:30,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:30,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:30,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:30,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153530274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:30,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:30,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:30,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:30,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153530274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:30,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:30,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:30,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:30,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:30,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:30,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T01:44:30,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T01:44:30,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:30,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:30,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:30,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:30,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:30,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:30,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T01:44:30,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:30,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:30,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:30,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:30,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:30,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:30,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/b53592f8387e46bb86b238df551b268b 2024-11-21T01:44:30,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/d4bac36c7bb7468f990eb3a9296a6b9c as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/d4bac36c7bb7468f990eb3a9296a6b9c 2024-11-21T01:44:30,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/d4bac36c7bb7468f990eb3a9296a6b9c, entries=200, sequenceid=52, filesize=38.6 K 2024-11-21T01:44:30,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/3eae683dcae342568f5bee745339d7e1 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/3eae683dcae342568f5bee745339d7e1 2024-11-21T01:44:30,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/3eae683dcae342568f5bee745339d7e1, entries=150, sequenceid=52, filesize=11.7 K 2024-11-21T01:44:30,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/b53592f8387e46bb86b238df551b268b as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/b53592f8387e46bb86b238df551b268b 2024-11-21T01:44:30,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/b53592f8387e46bb86b238df551b268b, entries=150, sequenceid=52, filesize=11.7 K 2024-11-21T01:44:30,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~135.62 KB/138872, heapSize ~388.19 KB/397504, currentSize=46.96 KB/48090 for 5835087c277e0e7bdbee836efeecc2f0 in 1035ms, sequenceid=52, compaction requested=false 2024-11-21T01:44:30,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:30,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T01:44:30,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:30,717 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 5835087c277e0e7bdbee836efeecc2f0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T01:44:30,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=A 2024-11-21T01:44:30,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:30,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=B 2024-11-21T01:44:30,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:30,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=C 2024-11-21T01:44:30,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:30,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f2b95a139d614a71a42a06d50c6c8540_5835087c277e0e7bdbee836efeecc2f0 is 50, key is test_row_0/A:col10/1732153469617/Put/seqid=0 2024-11-21T01:44:30,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:30,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:30,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741962_1138 (size=12154) 2024-11-21T01:44:30,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732153470808, max timestamp=1732153470831Num uniques 859; 2024-11-21T01:44:30,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732153470808, max timestamp=1732153470831Num uniques 859; 2024-11-21T01:44:30,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T01:44:30,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T01:44:30,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732153470808, max timestamp=1732153470831Num uniques 859; 2024-11-21T01:44:30,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T01:44:30,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:30,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42376 deadline: 1732153530868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:30,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:30,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:30,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:30,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:30,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153530870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:30,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:30,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:30,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:30,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:30,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153530872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:30,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:30,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153530873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:30,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:30,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:30,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:30,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:30,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153530875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:30,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:30,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:30,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:30,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:30,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42376 deadline: 1732153530983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:30,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:30,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:30,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:30,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153530983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:30,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:30,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:30,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:30,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:30,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153530983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:30,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:30,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:30,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:30,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:30,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:30,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153530984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:30,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:30,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153530984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:30,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:30,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:30,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:30,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:30,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:30,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:31,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:31,201 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f2b95a139d614a71a42a06d50c6c8540_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f2b95a139d614a71a42a06d50c6c8540_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:31,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/8fa01524a4554f9ea99178d70ad63255, store: [table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:31,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:31,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153531193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:31,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/8fa01524a4554f9ea99178d70ad63255 is 175, key is test_row_0/A:col10/1732153469617/Put/seqid=0 2024-11-21T01:44:31,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:31,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:31,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:31,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153531204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:31,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:31,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42376 deadline: 1732153531203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:31,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:31,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153531204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:31,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:31,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153531204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:31,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:31,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:31,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:31,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:31,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:31,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:31,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:31,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:31,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:31,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741963_1139 (size=30955) 2024-11-21T01:44:31,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153531517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:31,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:31,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:31,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:31,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42376 deadline: 1732153531523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:31,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:31,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153531524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:31,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:31,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:31,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:31,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:31,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153531524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:31,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:31,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:31,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:31,529 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:31,529 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:31,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:31,529 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:31,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153531524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:31,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:31,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:31,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:31,654 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/8fa01524a4554f9ea99178d70ad63255 2024-11-21T01:44:31,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/6f3fe1feb9cf47edac999792b9b1cf5c is 50, key is test_row_0/B:col10/1732153469617/Put/seqid=0 2024-11-21T01:44:31,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741964_1140 (size=12001) 2024-11-21T01:44:31,731 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/6f3fe1feb9cf47edac999792b9b1cf5c 2024-11-21T01:44:31,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/202db00ad7f943c2a76b0d517125f96e is 50, key is test_row_0/C:col10/1732153469617/Put/seqid=0 2024-11-21T01:44:31,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741965_1141 (size=12001) 2024-11-21T01:44:32,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:32,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153532033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:32,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:32,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:32,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:32,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:32,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153532034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:32,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:32,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:32,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:32,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:32,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153532043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:32,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:32,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:32,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153532044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:32,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:32,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:32,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:32,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:32,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:32,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42376 deadline: 1732153532044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:32,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:32,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:32,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:32,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:32,189 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/202db00ad7f943c2a76b0d517125f96e 2024-11-21T01:44:32,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/8fa01524a4554f9ea99178d70ad63255 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/8fa01524a4554f9ea99178d70ad63255 2024-11-21T01:44:32,302 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/8fa01524a4554f9ea99178d70ad63255, entries=150, sequenceid=62, filesize=30.2 K 2024-11-21T01:44:32,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/6f3fe1feb9cf47edac999792b9b1cf5c as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/6f3fe1feb9cf47edac999792b9b1cf5c 2024-11-21T01:44:32,327 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/6f3fe1feb9cf47edac999792b9b1cf5c, entries=150, sequenceid=62, filesize=11.7 K 2024-11-21T01:44:32,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/202db00ad7f943c2a76b0d517125f96e as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/202db00ad7f943c2a76b0d517125f96e 2024-11-21T01:44:32,335 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/202db00ad7f943c2a76b0d517125f96e, entries=150, sequenceid=62, filesize=11.7 K 2024-11-21T01:44:32,343 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 5835087c277e0e7bdbee836efeecc2f0 in 1626ms, sequenceid=62, compaction requested=true 2024-11-21T01:44:32,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:32,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:32,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-21T01:44:32,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-21T01:44:32,349 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-21T01:44:32,349 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.1310 sec 2024-11-21T01:44:32,364 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 4.1440 sec 2024-11-21T01:44:32,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T01:44:32,373 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:44:32,382 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:44:32,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-21T01:44:32,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T01:44:32,389 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:44:32,393 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:44:32,393 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:44:32,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T01:44:32,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T01:44:32,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:32,552 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 5835087c277e0e7bdbee836efeecc2f0 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-21T01:44:32,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=A 2024-11-21T01:44:32,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:32,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=B 2024-11-21T01:44:32,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:32,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=C 2024-11-21T01:44:32,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:32,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ed57002b59e14e31bfe58ce9a13cb93b_5835087c277e0e7bdbee836efeecc2f0 is 50, key is test_row_0/A:col10/1732153470873/Put/seqid=0 2024-11-21T01:44:32,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741966_1142 (size=12154) 2024-11-21T01:44:32,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T01:44:33,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T01:44:33,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:33,038 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ed57002b59e14e31bfe58ce9a13cb93b_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ed57002b59e14e31bfe58ce9a13cb93b_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:33,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/1f9666f2e8234a938d7120b10677e400, store: [table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:33,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/1f9666f2e8234a938d7120b10677e400 is 175, key is test_row_0/A:col10/1732153470873/Put/seqid=0 2024-11-21T01:44:33,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741967_1143 (size=30955) 2024-11-21T01:44:33,056 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/1f9666f2e8234a938d7120b10677e400 2024-11-21T01:44:33,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:33,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:33,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:33,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153533073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:33,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:33,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42376 deadline: 1732153533073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:33,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:33,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:33,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153533074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:33,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:33,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153533076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:33,084 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b93ada835ef8,41129,1732153434359 for 'test_row_1' in 5835087c277e0e7bdbee836efeecc2f0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T01:44:33,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:33,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153533077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:33,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:33,085 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,085 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,085 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:33,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:33,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:33,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/4e4e0a20063e4c56bedea8e85e4aaf8e is 50, key is test_row_0/B:col10/1732153470873/Put/seqid=0 2024-11-21T01:44:33,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741968_1144 (size=12001) 2024-11-21T01:44:33,147 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/4e4e0a20063e4c56bedea8e85e4aaf8e 2024-11-21T01:44:33,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/a1a4f8126c3f42cd9a788c091561328c is 50, key is test_row_0/C:col10/1732153470873/Put/seqid=0 2024-11-21T01:44:33,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:33,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153533183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:33,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:33,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741969_1145 (size=12001) 2024-11-21T01:44:33,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:33,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153533193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:33,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:33,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:33,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153533193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:33,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:33,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:33,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153533193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:33,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:33,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:33,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153533403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:33,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:33,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153533413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:33,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:33,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153533413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:33,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:33,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:33,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153533413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:33,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:33,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:33,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:33,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T01:44:33,604 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/a1a4f8126c3f42cd9a788c091561328c 2024-11-21T01:44:33,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/1f9666f2e8234a938d7120b10677e400 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/1f9666f2e8234a938d7120b10677e400 2024-11-21T01:44:33,620 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/1f9666f2e8234a938d7120b10677e400, entries=150, sequenceid=99, filesize=30.2 K 2024-11-21T01:44:33,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/4e4e0a20063e4c56bedea8e85e4aaf8e as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/4e4e0a20063e4c56bedea8e85e4aaf8e 2024-11-21T01:44:33,630 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/4e4e0a20063e4c56bedea8e85e4aaf8e, entries=150, sequenceid=99, filesize=11.7 K 2024-11-21T01:44:33,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/a1a4f8126c3f42cd9a788c091561328c as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/a1a4f8126c3f42cd9a788c091561328c 2024-11-21T01:44:33,651 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/a1a4f8126c3f42cd9a788c091561328c, entries=150, sequenceid=99, filesize=11.7 K 2024-11-21T01:44:33,653 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for 5835087c277e0e7bdbee836efeecc2f0 in 1100ms, sequenceid=99, compaction requested=true 2024-11-21T01:44:33,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:33,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:33,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-21T01:44:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-21T01:44:33,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-21T01:44:33,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2610 sec 2024-11-21T01:44:33,659 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.2750 sec 2024-11-21T01:44:33,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:33,729 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5835087c277e0e7bdbee836efeecc2f0 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T01:44:33,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=A 2024-11-21T01:44:33,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:33,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=B 2024-11-21T01:44:33,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:33,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=C 2024-11-21T01:44:33,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:33,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411217c945ae3a3e741448a87426967efc010_5835087c277e0e7bdbee836efeecc2f0 is 50, key is test_row_0/A:col10/1732153473074/Put/seqid=0 2024-11-21T01:44:33,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:33,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:33,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153533772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:33,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153533774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:33,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:33,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:33,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153533776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:33,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:33,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:33,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153533778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:33,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:33,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:33,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741970_1146 (size=12154) 2024-11-21T01:44:33,790 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:33,797 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411217c945ae3a3e741448a87426967efc010_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217c945ae3a3e741448a87426967efc010_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:33,798 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/4f4a27a074c94825bd8fd6dd6c4068dc, store: [table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:33,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/4f4a27a074c94825bd8fd6dd6c4068dc is 175, key is test_row_0/A:col10/1732153473074/Put/seqid=0 2024-11-21T01:44:33,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741971_1147 (size=30955) 2024-11-21T01:44:33,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:33,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153533893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:33,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:33,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153533894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:33,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:33,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153533894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:33,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:33,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153533894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:34,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:34,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153534104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:34,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:34,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:34,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:34,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:34,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153534106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:34,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:34,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153534106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:34,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:34,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:34,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:34,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:34,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:34,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:34,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:34,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153534114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:34,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:34,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:34,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:34,218 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=112, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/4f4a27a074c94825bd8fd6dd6c4068dc 2024-11-21T01:44:34,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/2a68c74403ea4a5aa5d9233c15a16104 is 50, key is test_row_0/B:col10/1732153473074/Put/seqid=0 2024-11-21T01:44:34,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741972_1148 (size=12001) 2024-11-21T01:44:34,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/2a68c74403ea4a5aa5d9233c15a16104 2024-11-21T01:44:34,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/4ce634183b064f2f8946c2ad90cbce7e is 50, key is test_row_0/C:col10/1732153473074/Put/seqid=0 2024-11-21T01:44:34,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741973_1149 (size=12001) 2024-11-21T01:44:34,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:34,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153534413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:34,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:34,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:34,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:34,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:34,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153534422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:34,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:34,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:34,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:34,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:34,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153534423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:34,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:34,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:34,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:34,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:34,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153534423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:34,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:34,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:34,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:34,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T01:44:34,537 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:44:34,538 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:44:34,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-21T01:44:34,540 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:44:34,541 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:44:34,541 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:44:34,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T01:44:34,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T01:44:34,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/4ce634183b064f2f8946c2ad90cbce7e 2024-11-21T01:44:34,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/4f4a27a074c94825bd8fd6dd6c4068dc as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/4f4a27a074c94825bd8fd6dd6c4068dc 2024-11-21T01:44:34,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T01:44:34,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:34,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:34,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:34,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:34,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:34,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:34,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/4f4a27a074c94825bd8fd6dd6c4068dc, entries=150, sequenceid=112, filesize=30.2 K 2024-11-21T01:44:34,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/2a68c74403ea4a5aa5d9233c15a16104 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/2a68c74403ea4a5aa5d9233c15a16104 2024-11-21T01:44:34,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/2a68c74403ea4a5aa5d9233c15a16104, entries=150, sequenceid=112, filesize=11.7 K 2024-11-21T01:44:34,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/4ce634183b064f2f8946c2ad90cbce7e as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/4ce634183b064f2f8946c2ad90cbce7e 2024-11-21T01:44:34,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/4ce634183b064f2f8946c2ad90cbce7e, entries=150, sequenceid=112, filesize=11.7 K 2024-11-21T01:44:34,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 5835087c277e0e7bdbee836efeecc2f0 in 1030ms, sequenceid=112, compaction requested=true 2024-11-21T01:44:34,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:34,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5835087c277e0e7bdbee836efeecc2f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:44:34,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:34,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5835087c277e0e7bdbee836efeecc2f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:44:34,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T01:44:34,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5835087c277e0e7bdbee836efeecc2f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:44:34,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-21T01:44:34,759 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-21T01:44:34,759 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-21T01:44:34,762 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 197737 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-21T01:44:34,762 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 5835087c277e0e7bdbee836efeecc2f0/A is initiating minor compaction (all files) 2024-11-21T01:44:34,762 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5835087c277e0e7bdbee836efeecc2f0/A in TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:34,762 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/a8fdcb23f6fa4860b229720a998831d1, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/d4bac36c7bb7468f990eb3a9296a6b9c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/8fa01524a4554f9ea99178d70ad63255, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/1f9666f2e8234a938d7120b10677e400, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/4f4a27a074c94825bd8fd6dd6c4068dc] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp, totalSize=193.1 K 2024-11-21T01:44:34,762 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:34,763 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. files: [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/a8fdcb23f6fa4860b229720a998831d1, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/d4bac36c7bb7468f990eb3a9296a6b9c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/8fa01524a4554f9ea99178d70ad63255, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/1f9666f2e8234a938d7120b10677e400, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/4f4a27a074c94825bd8fd6dd6c4068dc] 2024-11-21T01:44:34,763 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8fdcb23f6fa4860b229720a998831d1, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732153468211 2024-11-21T01:44:34,764 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4bac36c7bb7468f990eb3a9296a6b9c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732153468404 2024-11-21T01:44:34,765 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fa01524a4554f9ea99178d70ad63255, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732153469606 2024-11-21T01:44:34,765 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-21T01:44:34,765 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 5835087c277e0e7bdbee836efeecc2f0/C is initiating minor compaction (all files) 2024-11-21T01:44:34,765 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5835087c277e0e7bdbee836efeecc2f0/C in TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:34,765 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/4f3f4859f83142af94aab4c72c128fd0, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/b53592f8387e46bb86b238df551b268b, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/202db00ad7f943c2a76b0d517125f96e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/a1a4f8126c3f42cd9a788c091561328c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/4ce634183b064f2f8946c2ad90cbce7e] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp, totalSize=58.6 K 2024-11-21T01:44:34,766 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f9666f2e8234a938d7120b10677e400, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732153470861 2024-11-21T01:44:34,766 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f3f4859f83142af94aab4c72c128fd0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732153468224 2024-11-21T01:44:34,766 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f4a27a074c94825bd8fd6dd6c4068dc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732153473074 2024-11-21T01:44:34,767 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting b53592f8387e46bb86b238df551b268b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732153468415 2024-11-21T01:44:34,768 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 202db00ad7f943c2a76b0d517125f96e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732153469606 2024-11-21T01:44:34,768 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting a1a4f8126c3f42cd9a788c091561328c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732153470861 2024-11-21T01:44:34,769 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ce634183b064f2f8946c2ad90cbce7e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732153473074 2024-11-21T01:44:34,787 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:34,793 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5835087c277e0e7bdbee836efeecc2f0#C#compaction#130 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:34,794 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/1374b3fca7c14cddb446e9ede797cb38 is 50, key is test_row_0/C:col10/1732153473074/Put/seqid=0 2024-11-21T01:44:34,804 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411210a7f83e01bf6490989b2c76b1bf16aeb_5835087c277e0e7bdbee836efeecc2f0 store=[table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:34,817 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411210a7f83e01bf6490989b2c76b1bf16aeb_5835087c277e0e7bdbee836efeecc2f0, store=[table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:34,817 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210a7f83e01bf6490989b2c76b1bf16aeb_5835087c277e0e7bdbee836efeecc2f0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:34,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741974_1150 (size=12173) 2024-11-21T01:44:34,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T01:44:34,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:34,854 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 5835087c277e0e7bdbee836efeecc2f0 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T01:44:34,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=A 2024-11-21T01:44:34,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:34,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=B 2024-11-21T01:44:34,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:34,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=C 2024-11-21T01:44:34,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:34,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741975_1151 (size=4469) 2024-11-21T01:44:34,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T01:44:34,867 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/1374b3fca7c14cddb446e9ede797cb38 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/1374b3fca7c14cddb446e9ede797cb38 2024-11-21T01:44:34,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ad5dff4c5d254113845009ba857f0dcc_5835087c277e0e7bdbee836efeecc2f0 is 50, key is test_row_0/A:col10/1732153473774/Put/seqid=0 2024-11-21T01:44:34,878 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5835087c277e0e7bdbee836efeecc2f0/C of 5835087c277e0e7bdbee836efeecc2f0 into 1374b3fca7c14cddb446e9ede797cb38(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:34,878 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:34,878 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., storeName=5835087c277e0e7bdbee836efeecc2f0/C, priority=11, startTime=1732153474759; duration=0sec 2024-11-21T01:44:34,878 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:34,878 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5835087c277e0e7bdbee836efeecc2f0:C 2024-11-21T01:44:34,878 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-21T01:44:34,880 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-21T01:44:34,880 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 5835087c277e0e7bdbee836efeecc2f0/B is initiating minor compaction (all files) 2024-11-21T01:44:34,880 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5835087c277e0e7bdbee836efeecc2f0/B in TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:34,880 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/70e8711ecfaf4b7992f91cfce9e4743f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/3eae683dcae342568f5bee745339d7e1, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/6f3fe1feb9cf47edac999792b9b1cf5c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/4e4e0a20063e4c56bedea8e85e4aaf8e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/2a68c74403ea4a5aa5d9233c15a16104] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp, totalSize=58.6 K 2024-11-21T01:44:34,880 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 70e8711ecfaf4b7992f91cfce9e4743f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732153468224 2024-11-21T01:44:34,881 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 3eae683dcae342568f5bee745339d7e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732153468415 2024-11-21T01:44:34,881 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f3fe1feb9cf47edac999792b9b1cf5c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732153469606 2024-11-21T01:44:34,882 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e4e0a20063e4c56bedea8e85e4aaf8e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732153470861 2024-11-21T01:44:34,883 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a68c74403ea4a5aa5d9233c15a16104, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732153473074 2024-11-21T01:44:34,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741976_1152 (size=12254) 2024-11-21T01:44:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:34,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:34,939 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5835087c277e0e7bdbee836efeecc2f0#B#compaction#132 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:34,943 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/eca8558978b4455fb5aff4abfef0f8f1 is 50, key is test_row_0/B:col10/1732153473074/Put/seqid=0 2024-11-21T01:44:34,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741977_1153 (size=12173) 2024-11-21T01:44:34,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:34,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153534978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:34,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:34,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153534982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:34,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:34,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:34,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:34,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:34,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:34,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:34,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153534984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:34,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:34,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:34,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:34,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153534987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:34,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:34,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:34,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:34,996 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/eca8558978b4455fb5aff4abfef0f8f1 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/eca8558978b4455fb5aff4abfef0f8f1 2024-11-21T01:44:35,007 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5835087c277e0e7bdbee836efeecc2f0/B of 5835087c277e0e7bdbee836efeecc2f0 into eca8558978b4455fb5aff4abfef0f8f1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:35,007 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:35,007 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., storeName=5835087c277e0e7bdbee836efeecc2f0/B, priority=11, startTime=1732153474759; duration=0sec 2024-11-21T01:44:35,007 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:35,007 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5835087c277e0e7bdbee836efeecc2f0:B 2024-11-21T01:44:35,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:35,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:35,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153535104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:35,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153535104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:35,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:35,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:35,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153535105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:35,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:35,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:35,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:35,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:35,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153535106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:35,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:35,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:35,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:35,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:35,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42376 deadline: 1732153535113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:35,119 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b93ada835ef8,41129,1732153434359 for 'test_row_1' in 5835087c277e0e7bdbee836efeecc2f0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4251 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T01:44:35,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:35,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:35,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:35,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T01:44:35,269 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5835087c277e0e7bdbee836efeecc2f0#A#compaction#129 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:35,270 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/4519ef66825c49c8ab6dee31a66bc435 is 175, key is test_row_0/A:col10/1732153473074/Put/seqid=0 2024-11-21T01:44:35,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741978_1154 (size=31127) 2024-11-21T01:44:35,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:35,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153535313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:35,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:35,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153535313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:35,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:35,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153535313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:35,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:35,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153535313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:35,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:35,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:35,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:35,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:35,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:35,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:35,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:35,325 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ad5dff4c5d254113845009ba857f0dcc_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ad5dff4c5d254113845009ba857f0dcc_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:35,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/12ec4d42fd1845e5a16d7852fa976dde, store: [table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:35,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/12ec4d42fd1845e5a16d7852fa976dde is 175, key is test_row_0/A:col10/1732153473774/Put/seqid=0 2024-11-21T01:44:35,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741979_1155 (size=31055) 2024-11-21T01:44:35,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:35,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153535623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:35,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:35,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:35,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:35,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153535623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:35,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:35,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:35,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:35,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:35,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153535624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:35,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153535624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:35,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:35,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:35,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:35,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:35,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:35,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T01:44:35,684 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/4519ef66825c49c8ab6dee31a66bc435 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/4519ef66825c49c8ab6dee31a66bc435 2024-11-21T01:44:35,695 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5835087c277e0e7bdbee836efeecc2f0/A of 5835087c277e0e7bdbee836efeecc2f0 into 4519ef66825c49c8ab6dee31a66bc435(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:35,695 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:35,695 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., storeName=5835087c277e0e7bdbee836efeecc2f0/A, priority=11, startTime=1732153474758; duration=0sec 2024-11-21T01:44:35,696 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:35,696 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5835087c277e0e7bdbee836efeecc2f0:A 2024-11-21T01:44:35,735 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/12ec4d42fd1845e5a16d7852fa976dde 2024-11-21T01:44:35,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/298dfd1916af4b248bcfbab29e595e03 is 50, key is test_row_0/B:col10/1732153473774/Put/seqid=0 2024-11-21T01:44:35,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741980_1156 (size=12101) 2024-11-21T01:44:35,796 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/298dfd1916af4b248bcfbab29e595e03 2024-11-21T01:44:35,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/dc9502f1084d423195875dfbfe704095 is 50, key is test_row_0/C:col10/1732153473774/Put/seqid=0 2024-11-21T01:44:35,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741981_1157 (size=12101) 2024-11-21T01:44:36,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:36,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153536133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:36,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:36,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:36,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:36,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:36,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153536133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:36,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:36,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153536133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:36,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:36,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:36,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:36,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:36,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:36,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:36,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:36,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153536143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:36,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:36,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:36,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:36,235 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/dc9502f1084d423195875dfbfe704095 2024-11-21T01:44:36,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/12ec4d42fd1845e5a16d7852fa976dde as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/12ec4d42fd1845e5a16d7852fa976dde 2024-11-21T01:44:36,277 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/12ec4d42fd1845e5a16d7852fa976dde, entries=150, sequenceid=131, filesize=30.3 K 2024-11-21T01:44:36,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/298dfd1916af4b248bcfbab29e595e03 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/298dfd1916af4b248bcfbab29e595e03 2024-11-21T01:44:36,285 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/298dfd1916af4b248bcfbab29e595e03, entries=150, sequenceid=131, filesize=11.8 K 2024-11-21T01:44:36,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/dc9502f1084d423195875dfbfe704095 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/dc9502f1084d423195875dfbfe704095 2024-11-21T01:44:36,311 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/dc9502f1084d423195875dfbfe704095, entries=150, sequenceid=131, filesize=11.8 K 2024-11-21T01:44:36,316 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 5835087c277e0e7bdbee836efeecc2f0 in 1462ms, sequenceid=131, compaction requested=false 2024-11-21T01:44:36,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:36,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:36,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-21T01:44:36,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-21T01:44:36,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-21T01:44:36,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7770 sec 2024-11-21T01:44:36,322 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.7830 sec 2024-11-21T01:44:36,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T01:44:36,693 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:44:36,695 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:44:36,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-21T01:44:36,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T01:44:36,701 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:44:36,705 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:44:36,705 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:44:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T01:44:36,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T01:44:36,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:36,879 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 5835087c277e0e7bdbee836efeecc2f0 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T01:44:36,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=A 2024-11-21T01:44:36,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:36,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=B 2024-11-21T01:44:36,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:36,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=C 2024-11-21T01:44:36,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:36,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ea483e4f66484ebaa88eb84563253095_5835087c277e0e7bdbee836efeecc2f0 is 50, key is test_row_0/A:col10/1732153474926/Put/seqid=0 2024-11-21T01:44:36,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741982_1158 (size=12304) 2024-11-21T01:44:36,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:36,935 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ea483e4f66484ebaa88eb84563253095_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ea483e4f66484ebaa88eb84563253095_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:36,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/abdef8ee57e0439a9fd03a53fe158c7d, store: [table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:36,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/abdef8ee57e0439a9fd03a53fe158c7d is 175, key is test_row_0/A:col10/1732153474926/Put/seqid=0 2024-11-21T01:44:36,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741983_1159 (size=31105) 2024-11-21T01:44:36,945 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=147, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/abdef8ee57e0439a9fd03a53fe158c7d 2024-11-21T01:44:36,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/746775a3eac546cda44f1e53195e7098 is 50, key is test_row_0/B:col10/1732153474926/Put/seqid=0 2024-11-21T01:44:37,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741984_1160 (size=12151) 2024-11-21T01:44:37,006 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/746775a3eac546cda44f1e53195e7098 2024-11-21T01:44:37,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T01:44:37,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/d7b522a8644b401aac51ba34e69938b4 is 50, key is test_row_0/C:col10/1732153474926/Put/seqid=0 2024-11-21T01:44:37,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741985_1161 (size=12151) 2024-11-21T01:44:37,049 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/d7b522a8644b401aac51ba34e69938b4 2024-11-21T01:44:37,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/abdef8ee57e0439a9fd03a53fe158c7d as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/abdef8ee57e0439a9fd03a53fe158c7d 2024-11-21T01:44:37,076 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/abdef8ee57e0439a9fd03a53fe158c7d, entries=150, sequenceid=147, filesize=30.4 K 2024-11-21T01:44:37,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/746775a3eac546cda44f1e53195e7098 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/746775a3eac546cda44f1e53195e7098 2024-11-21T01:44:37,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,084 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/746775a3eac546cda44f1e53195e7098, entries=150, sequenceid=147, filesize=11.9 K 2024-11-21T01:44:37,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/d7b522a8644b401aac51ba34e69938b4 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/d7b522a8644b401aac51ba34e69938b4 2024-11-21T01:44:37,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,090 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/d7b522a8644b401aac51ba34e69938b4, entries=150, sequenceid=147, filesize=11.9 K 2024-11-21T01:44:37,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,091 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 5835087c277e0e7bdbee836efeecc2f0 in 212ms, sequenceid=147, compaction requested=true 2024-11-21T01:44:37,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:37,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:37,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-21T01:44:37,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-21T01:44:37,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,097 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-21T01:44:37,097 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 389 msec 2024-11-21T01:44:37,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,099 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 403 msec 2024-11-21T01:44:37,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:37,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5835087c277e0e7bdbee836efeecc2f0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T01:44:37,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=A 2024-11-21T01:44:37,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:37,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=B 2024-11-21T01:44:37,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:37,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=C 2024-11-21T01:44:37,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219c7ac4050abd4a05917bed57409da7ce_5835087c277e0e7bdbee836efeecc2f0 is 50, key is test_row_0/A:col10/1732153477211/Put/seqid=0 2024-11-21T01:44:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1732153477281, max timestamp=1732153477308Num uniques 859; 2024-11-21T01:44:37,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1732153477281, max timestamp=1732153477308Num uniques 859; 2024-11-21T01:44:37,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-21T01:44:37,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-21T01:44:37,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1732153477281, max timestamp=1732153477308Num uniques 859; 2024-11-21T01:44:37,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-21T01:44:37,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741986_1162 (size=17284) 2024-11-21T01:44:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T01:44:37,323 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:44:37,325 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:44:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-21T01:44:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T01:44:37,327 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:44:37,328 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:44:37,328 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:44:37,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:37,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153537335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:37,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:37,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:37,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:37,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:37,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153537351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:37,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:37,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153537352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:37,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153537353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:37,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T01:44:37,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:37,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153537466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:37,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:37,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:37,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:37,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:37,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153537473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:37,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:37,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:37,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:37,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:37,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153537473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:37,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:37,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153537474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:37,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:37,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:37,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:37,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:37,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:37,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:37,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T01:44:37,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:37,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:37,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:37,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:37,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:37,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:37,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T01:44:37,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:37,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:37,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:37,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:37,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:37,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:37,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T01:44:37,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:37,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153537685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:37,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:37,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:37,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153537685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:37,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:37,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:37,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:37,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:37,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:37,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:37,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153537693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:37,699 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:37,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:37,699 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:37,699 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:37,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153537693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:37,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:37,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:37,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:37,722 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:37,727 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219c7ac4050abd4a05917bed57409da7ce_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219c7ac4050abd4a05917bed57409da7ce_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:37,729 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/acd27e1d2bc1479eb8b26424365dc0a2, store: [table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:37,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/acd27e1d2bc1479eb8b26424365dc0a2 is 175, key is test_row_0/A:col10/1732153477211/Put/seqid=0 2024-11-21T01:44:37,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741987_1163 (size=48389) 2024-11-21T01:44:37,737 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/acd27e1d2bc1479eb8b26424365dc0a2 2024-11-21T01:44:37,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/5ac6db6aa8a24268aecd5c4a0f9888b3 is 50, key is test_row_0/B:col10/1732153477211/Put/seqid=0 2024-11-21T01:44:37,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741988_1164 (size=12151) 2024-11-21T01:44:37,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T01:44:37,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:37,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:37,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:37,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:37,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:37,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:37,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T01:44:37,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:37,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:37,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:37,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:37,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:37,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T01:44:37,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153537993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:37,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:37,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:37,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153537993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:37,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:37,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:37,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:37,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:38,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:38,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153538003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:38,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:38,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:38,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:38,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:38,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153538003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:38,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:38,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:38,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:38,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T01:44:38,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:38,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:38,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:38,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:38,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:38,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:38,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/5ac6db6aa8a24268aecd5c4a0f9888b3 2024-11-21T01:44:38,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/201f9b8bd81b41fbbdff14fbd90de59f is 50, key is test_row_0/C:col10/1732153477211/Put/seqid=0 2024-11-21T01:44:38,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741989_1165 (size=12151) 2024-11-21T01:44:38,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/201f9b8bd81b41fbbdff14fbd90de59f 2024-11-21T01:44:38,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/acd27e1d2bc1479eb8b26424365dc0a2 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/acd27e1d2bc1479eb8b26424365dc0a2 2024-11-21T01:44:38,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/acd27e1d2bc1479eb8b26424365dc0a2, entries=250, sequenceid=157, filesize=47.3 K 2024-11-21T01:44:38,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/5ac6db6aa8a24268aecd5c4a0f9888b3 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/5ac6db6aa8a24268aecd5c4a0f9888b3 2024-11-21T01:44:38,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/5ac6db6aa8a24268aecd5c4a0f9888b3, entries=150, sequenceid=157, filesize=11.9 K 2024-11-21T01:44:38,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/201f9b8bd81b41fbbdff14fbd90de59f as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/201f9b8bd81b41fbbdff14fbd90de59f 2024-11-21T01:44:38,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/201f9b8bd81b41fbbdff14fbd90de59f, entries=150, sequenceid=157, filesize=11.9 K 2024-11-21T01:44:38,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 5835087c277e0e7bdbee836efeecc2f0 in 994ms, sequenceid=157, compaction requested=true 2024-11-21T01:44:38,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:38,249 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:44:38,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5835087c277e0e7bdbee836efeecc2f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:44:38,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:38,249 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:44:38,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5835087c277e0e7bdbee836efeecc2f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:44:38,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:38,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5835087c277e0e7bdbee836efeecc2f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:44:38,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:38,251 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141676 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:44:38,251 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 5835087c277e0e7bdbee836efeecc2f0/A is initiating minor compaction (all files) 2024-11-21T01:44:38,251 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5835087c277e0e7bdbee836efeecc2f0/A in TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:38,251 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/4519ef66825c49c8ab6dee31a66bc435, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/12ec4d42fd1845e5a16d7852fa976dde, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/abdef8ee57e0439a9fd03a53fe158c7d, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/acd27e1d2bc1479eb8b26424365dc0a2] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp, totalSize=138.4 K 2024-11-21T01:44:38,251 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:38,251 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. files: [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/4519ef66825c49c8ab6dee31a66bc435, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/12ec4d42fd1845e5a16d7852fa976dde, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/abdef8ee57e0439a9fd03a53fe158c7d, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/acd27e1d2bc1479eb8b26424365dc0a2] 2024-11-21T01:44:38,252 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4519ef66825c49c8ab6dee31a66bc435, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732153473074 2024-11-21T01:44:38,252 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48576 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:44:38,252 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 5835087c277e0e7bdbee836efeecc2f0/B is initiating minor compaction (all files) 2024-11-21T01:44:38,253 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5835087c277e0e7bdbee836efeecc2f0/B in TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:38,253 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/eca8558978b4455fb5aff4abfef0f8f1, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/298dfd1916af4b248bcfbab29e595e03, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/746775a3eac546cda44f1e53195e7098, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/5ac6db6aa8a24268aecd5c4a0f9888b3] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp, totalSize=47.4 K 2024-11-21T01:44:38,253 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12ec4d42fd1845e5a16d7852fa976dde, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732153473772 2024-11-21T01:44:38,253 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting eca8558978b4455fb5aff4abfef0f8f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732153473074 2024-11-21T01:44:38,254 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 298dfd1916af4b248bcfbab29e595e03, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732153473772 2024-11-21T01:44:38,254 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting abdef8ee57e0439a9fd03a53fe158c7d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732153474926 2024-11-21T01:44:38,254 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 746775a3eac546cda44f1e53195e7098, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732153474926 2024-11-21T01:44:38,254 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting acd27e1d2bc1479eb8b26424365dc0a2, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732153477210 2024-11-21T01:44:38,255 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ac6db6aa8a24268aecd5c4a0f9888b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732153477210 2024-11-21T01:44:38,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T01:44:38,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:38,273 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 5835087c277e0e7bdbee836efeecc2f0 3/3 column families, dataSize=140.89 KB heapSize=376.78 KB 2024-11-21T01:44:38,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=A 2024-11-21T01:44:38,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:38,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=B 2024-11-21T01:44:38,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:38,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=C 2024-11-21T01:44:38,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:38,284 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5835087c277e0e7bdbee836efeecc2f0#B#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:38,285 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/1e8a5d259c2b4050a5a3145b054fa602 is 50, key is test_row_0/B:col10/1732153477211/Put/seqid=0 2024-11-21T01:44:38,290 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:38,306 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411219f21e51f68174d13ab2fbae67029dfec_5835087c277e0e7bdbee836efeecc2f0 store=[table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:38,309 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411219f21e51f68174d13ab2fbae67029dfec_5835087c277e0e7bdbee836efeecc2f0, store=[table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:38,310 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219f21e51f68174d13ab2fbae67029dfec_5835087c277e0e7bdbee836efeecc2f0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:38,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121567b7da6ae934287baf53b924940e836_5835087c277e0e7bdbee836efeecc2f0 is 50, key is test_row_0/A:col10/1732153477351/Put/seqid=0 2024-11-21T01:44:38,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741990_1166 (size=12459) 2024-11-21T01:44:38,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741991_1167 (size=4469) 2024-11-21T01:44:38,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741992_1168 (size=12304) 2024-11-21T01:44:38,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:38,338 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5835087c277e0e7bdbee836efeecc2f0#A#compaction#142 average throughput is 0.51 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:38,339 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/bc6b7ba3300a488da3ba1b9ab5e8c88d is 175, key is test_row_0/A:col10/1732153477211/Put/seqid=0 2024-11-21T01:44:38,342 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121567b7da6ae934287baf53b924940e836_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121567b7da6ae934287baf53b924940e836_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:38,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/332657bf6e8d473dae18570fe91864c4, store: [table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:38,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/332657bf6e8d473dae18570fe91864c4 is 175, key is test_row_0/A:col10/1732153477351/Put/seqid=0 2024-11-21T01:44:38,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741993_1169 (size=31413) 2024-11-21T01:44:38,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741994_1170 (size=31105) 2024-11-21T01:44:38,359 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/1e8a5d259c2b4050a5a3145b054fa602 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/1e8a5d259c2b4050a5a3145b054fa602 2024-11-21T01:44:38,367 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5835087c277e0e7bdbee836efeecc2f0/B of 5835087c277e0e7bdbee836efeecc2f0 into 1e8a5d259c2b4050a5a3145b054fa602(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:38,367 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:38,367 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., storeName=5835087c277e0e7bdbee836efeecc2f0/B, priority=12, startTime=1732153478249; duration=0sec 2024-11-21T01:44:38,367 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:38,367 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5835087c277e0e7bdbee836efeecc2f0:B 2024-11-21T01:44:38,367 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:44:38,369 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48576 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:44:38,369 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 5835087c277e0e7bdbee836efeecc2f0/C is initiating minor compaction (all files) 2024-11-21T01:44:38,369 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5835087c277e0e7bdbee836efeecc2f0/C in TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:38,369 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/1374b3fca7c14cddb446e9ede797cb38, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/dc9502f1084d423195875dfbfe704095, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/d7b522a8644b401aac51ba34e69938b4, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/201f9b8bd81b41fbbdff14fbd90de59f] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp, totalSize=47.4 K 2024-11-21T01:44:38,370 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 1374b3fca7c14cddb446e9ede797cb38, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732153473074 2024-11-21T01:44:38,370 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting dc9502f1084d423195875dfbfe704095, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732153473772 2024-11-21T01:44:38,371 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting d7b522a8644b401aac51ba34e69938b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732153474926 2024-11-21T01:44:38,371 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 201f9b8bd81b41fbbdff14fbd90de59f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732153477210 2024-11-21T01:44:38,405 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5835087c277e0e7bdbee836efeecc2f0#C#compaction#144 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:38,405 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/38869b7de0df4f66ac09aa35dd3dd0dc is 50, key is test_row_0/C:col10/1732153477211/Put/seqid=0 2024-11-21T01:44:38,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741995_1171 (size=12459) 2024-11-21T01:44:38,417 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/38869b7de0df4f66ac09aa35dd3dd0dc as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/38869b7de0df4f66ac09aa35dd3dd0dc 2024-11-21T01:44:38,422 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5835087c277e0e7bdbee836efeecc2f0/C of 5835087c277e0e7bdbee836efeecc2f0 into 38869b7de0df4f66ac09aa35dd3dd0dc(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:38,422 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:38,422 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., storeName=5835087c277e0e7bdbee836efeecc2f0/C, priority=12, startTime=1732153478249; duration=0sec 2024-11-21T01:44:38,422 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:38,422 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5835087c277e0e7bdbee836efeecc2f0:C 2024-11-21T01:44:38,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T01:44:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:38,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:38,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:38,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153538538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:38,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:38,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:38,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:38,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153538539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:38,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:38,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:38,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:38,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153538540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:38,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:38,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:38,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153538540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:38,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:38,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:38,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:38,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:38,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:38,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153538654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:38,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:38,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:38,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:38,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153538655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:38,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:38,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:38,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:38,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153538655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:38,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:38,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:38,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:38,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153538658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:38,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:38,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:38,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:38,753 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=189, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/332657bf6e8d473dae18570fe91864c4 2024-11-21T01:44:38,758 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/bc6b7ba3300a488da3ba1b9ab5e8c88d as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/bc6b7ba3300a488da3ba1b9ab5e8c88d 2024-11-21T01:44:38,769 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5835087c277e0e7bdbee836efeecc2f0/A of 5835087c277e0e7bdbee836efeecc2f0 into bc6b7ba3300a488da3ba1b9ab5e8c88d(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:38,769 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:38,769 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., storeName=5835087c277e0e7bdbee836efeecc2f0/A, priority=12, startTime=1732153478249; duration=0sec 2024-11-21T01:44:38,770 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:38,770 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5835087c277e0e7bdbee836efeecc2f0:A 2024-11-21T01:44:38,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/a6fc4714f7ce474b8c82134eba1da198 is 50, key is test_row_0/B:col10/1732153477351/Put/seqid=0 2024-11-21T01:44:38,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741996_1172 (size=12151) 2024-11-21T01:44:38,814 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/a6fc4714f7ce474b8c82134eba1da198 2024-11-21T01:44:38,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/6405a90cc0cb417280b11a4a3ff2236f is 50, key is test_row_0/C:col10/1732153477351/Put/seqid=0 2024-11-21T01:44:38,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741997_1173 (size=12151) 2024-11-21T01:44:38,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:38,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153538863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:38,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:38,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:38,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:38,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:38,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:38,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153538863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:38,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153538864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:38,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:38,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:38,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:38,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:38,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:38,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:38,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:38,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153538872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:38,886 INFO [master/b93ada835ef8:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-21T01:44:38,886 INFO [master/b93ada835ef8:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-21T01:44:39,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:39,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42376 deadline: 1732153539153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:39,157 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b93ada835ef8,41129,1732153434359 for 'test_row_1' in 5835087c277e0e7bdbee836efeecc2f0 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8289 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T01:44:39,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:39,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:39,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:39,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:39,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153539173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:39,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:39,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:39,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:39,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:39,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153539183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:39,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:39,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:39,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:39,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153539183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:39,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:39,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:39,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:39,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:39,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:39,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153539183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:39,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:39,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:39,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:39,258 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/6405a90cc0cb417280b11a4a3ff2236f 2024-11-21T01:44:39,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/332657bf6e8d473dae18570fe91864c4 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/332657bf6e8d473dae18570fe91864c4 2024-11-21T01:44:39,268 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/332657bf6e8d473dae18570fe91864c4, entries=150, sequenceid=189, filesize=30.4 K 2024-11-21T01:44:39,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/a6fc4714f7ce474b8c82134eba1da198 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/a6fc4714f7ce474b8c82134eba1da198 2024-11-21T01:44:39,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,282 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/a6fc4714f7ce474b8c82134eba1da198, entries=150, sequenceid=189, filesize=11.9 K 2024-11-21T01:44:39,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/6405a90cc0cb417280b11a4a3ff2236f as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/6405a90cc0cb417280b11a4a3ff2236f 2024-11-21T01:44:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,294 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/6405a90cc0cb417280b11a4a3ff2236f, entries=150, sequenceid=189, filesize=11.9 K 2024-11-21T01:44:39,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,295 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~376.73 KB/385776, currentSize=46.96 KB/48090 for 5835087c277e0e7bdbee836efeecc2f0 in 1022ms, sequenceid=189, compaction requested=false 2024-11-21T01:44:39,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:39,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:39,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-21T01:44:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-21T01:44:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,299 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-21T01:44:39,299 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9690 sec 2024-11-21T01:44:39,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,301 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.9750 sec 2024-11-21T01:44:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T01:44:39,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,474 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:44:39,474 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:44:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-21T01:44:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,477 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:44:39,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,478 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:44:39,478 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:44:39,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T01:44:39,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T01:44:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-21T01:44:39,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:39,635 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 5835087c277e0e7bdbee836efeecc2f0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T01:44:39,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=A 2024-11-21T01:44:39,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:39,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=B 2024-11-21T01:44:39,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:39,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=C 2024-11-21T01:44:39,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:39,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215b1dbb0803604e1690e2dfe6b53e03e8_5835087c277e0e7bdbee836efeecc2f0 is 50, key is test_row_1/A:col10/1732153478537/Put/seqid=0 2024-11-21T01:44:39,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741998_1174 (size=9814) 2024-11-21T01:44:39,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,694 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215b1dbb0803604e1690e2dfe6b53e03e8_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215b1dbb0803604e1690e2dfe6b53e03e8_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:39,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/6a99845586d34d51b99a332aa8c78e10, store: [table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:39,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/6a99845586d34d51b99a332aa8c78e10 is 175, key is test_row_1/A:col10/1732153478537/Put/seqid=0 2024-11-21T01:44:39,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:39,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:39,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:39,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741999_1175 (size=22461) 2024-11-21T01:44:39,744 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=202, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/6a99845586d34d51b99a332aa8c78e10 2024-11-21T01:44:39,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/5f07c54247bf4267b1068b0a865497f4 is 50, key is test_row_1/B:col10/1732153478537/Put/seqid=0 2024-11-21T01:44:39,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732153479730, max timestamp=1732153479775Num uniques 859; 2024-11-21T01:44:39,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-21T01:44:39,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732153479730, max timestamp=1732153479775Num uniques 859; 2024-11-21T01:44:39,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-21T01:44:39,786 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732153479730, max timestamp=1732153479775Num uniques 859; 2024-11-21T01:44:39,787 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-21T01:44:39,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T01:44:39,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742000_1176 (size=9757) 2024-11-21T01:44:39,804 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/5f07c54247bf4267b1068b0a865497f4 2024-11-21T01:44:39,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/40650b2bbd2141c9a84c9398d151c210 is 50, key is test_row_1/C:col10/1732153478537/Put/seqid=0 2024-11-21T01:44:39,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153539834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:39,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153539840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:39,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:39,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:39,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:39,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:39,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:39,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:39,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:39,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153539848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:39,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:39,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:39,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:39,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:39,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153539848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:39,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:39,851 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:39,851 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:39,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742001_1177 (size=9757) 2024-11-21T01:44:39,865 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/40650b2bbd2141c9a84c9398d151c210 2024-11-21T01:44:39,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/6a99845586d34d51b99a332aa8c78e10 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/6a99845586d34d51b99a332aa8c78e10 2024-11-21T01:44:39,881 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/6a99845586d34d51b99a332aa8c78e10, entries=100, sequenceid=202, filesize=21.9 K 2024-11-21T01:44:39,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/5f07c54247bf4267b1068b0a865497f4 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/5f07c54247bf4267b1068b0a865497f4 2024-11-21T01:44:39,891 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/5f07c54247bf4267b1068b0a865497f4, entries=100, sequenceid=202, filesize=9.5 K 2024-11-21T01:44:39,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/40650b2bbd2141c9a84c9398d151c210 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/40650b2bbd2141c9a84c9398d151c210 2024-11-21T01:44:39,898 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/40650b2bbd2141c9a84c9398d151c210, entries=100, sequenceid=202, filesize=9.5 K 2024-11-21T01:44:39,900 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for 5835087c277e0e7bdbee836efeecc2f0 in 265ms, sequenceid=202, compaction requested=true 2024-11-21T01:44:39,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:39,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:39,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-21T01:44:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-21T01:44:39,904 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-21T01:44:39,904 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 424 msec 2024-11-21T01:44:39,906 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 431 msec 2024-11-21T01:44:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:39,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5835087c277e0e7bdbee836efeecc2f0 3/3 column families, dataSize=154.31 KB heapSize=431.39 KB 2024-11-21T01:44:39,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732153479827, max timestamp=1732153479962Num uniques 1168; 2024-11-21T01:44:39,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=9978, suffix heap size=149440,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=218 2024-11-21T01:44:39,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732153479827, max timestamp=1732153479962Num uniques 1168; 2024-11-21T01:44:39,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=9978, suffix heap size=149440,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=218 2024-11-21T01:44:39,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=A 2024-11-21T01:44:39,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:39,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=B 2024-11-21T01:44:39,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732153479827, max timestamp=1732153479962Num uniques 1168; 2024-11-21T01:44:39,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:39,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=C 2024-11-21T01:44:39,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:39,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=9978, suffix heap size=149440,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=218 2024-11-21T01:44:39,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f3fdd3e444bc48cf9fdb3321dc561b09_5835087c277e0e7bdbee836efeecc2f0 is 50, key is test_row_0/A:col10/1732153479827/Put/seqid=0 2024-11-21T01:44:40,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732153479985, max timestamp=1732153480013Num uniques 859; 2024-11-21T01:44:40,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-21T01:44:40,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732153479985, max timestamp=1732153480013Num uniques 859; 2024-11-21T01:44:40,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-21T01:44:40,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732153479985, max timestamp=1732153480013Num uniques 859; 2024-11-21T01:44:40,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-21T01:44:40,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742002_1178 (size=12304) 2024-11-21T01:44:40,031 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:40,038 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f3fdd3e444bc48cf9fdb3321dc561b09_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f3fdd3e444bc48cf9fdb3321dc561b09_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:40,039 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/89accf3e359d4f788f11bf2a357dd151, store: [table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:40,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/89accf3e359d4f788f11bf2a357dd151 is 175, key is test_row_0/A:col10/1732153479827/Put/seqid=0 2024-11-21T01:44:40,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742003_1179 (size=31105) 2024-11-21T01:44:40,070 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=240, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/89accf3e359d4f788f11bf2a357dd151 2024-11-21T01:44:40,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=568, cellSize=26008, totalHeapSize=59312, min timestamp=1732153480009, max timestamp=1732153480081Num uniques 1168; 2024-11-21T01:44:40,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=26008, suffix heap size=149440,new segment heap size=59312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=568 2024-11-21T01:44:40,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=568, cellSize=26008, totalHeapSize=59312, min timestamp=1732153480009, max timestamp=1732153480084Num uniques 1168; 2024-11-21T01:44:40,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=568, cellSize=26008, totalHeapSize=59312, min timestamp=1732153480009, max timestamp=1732153480084Num uniques 1168; 2024-11-21T01:44:40,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=26008, suffix heap size=149440,new segment heap size=59312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=568 2024-11-21T01:44:40,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=26008, suffix heap size=149440,new segment heap size=59312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=568 2024-11-21T01:44:40,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/81315e934a6b42c1818d40bed066090a is 50, key is test_row_0/B:col10/1732153479827/Put/seqid=0 2024-11-21T01:44:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T01:44:40,103 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:44:40,104 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:44:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-21T01:44:40,106 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:44:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T01:44:40,107 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:44:40,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:44:40,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742004_1180 (size=12151) 2024-11-21T01:44:40,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/81315e934a6b42c1818d40bed066090a 2024-11-21T01:44:40,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153540116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:40,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:40,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153540117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:40,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:40,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153540117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:40,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:40,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:40,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153540120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:40,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:40,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/cbc60e7b87494fc0bc4f38595c5562d9 is 50, key is test_row_0/C:col10/1732153479827/Put/seqid=0 2024-11-21T01:44:40,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742005_1181 (size=12151) 2024-11-21T01:44:40,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T01:44:40,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:40,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153540233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:40,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:40,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153540234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:40,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:40,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:40,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153540234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:40,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:40,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153540234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:40,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:40,237 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:40,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T01:44:40,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:40,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:40,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:40,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:40,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:40,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:40,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T01:44:40,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:40,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:40,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:40,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:40,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:40,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:40,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T01:44:40,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:40,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153540442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:40,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:40,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:40,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153540443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:40,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:40,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153540444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:40,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:40,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:40,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:40,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153540444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:40,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:40,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/cbc60e7b87494fc0bc4f38595c5562d9 2024-11-21T01:44:40,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/89accf3e359d4f788f11bf2a357dd151 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/89accf3e359d4f788f11bf2a357dd151 2024-11-21T01:44:40,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T01:44:40,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:40,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:40,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:40,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:40,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:40,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:40,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/89accf3e359d4f788f11bf2a357dd151, entries=150, sequenceid=240, filesize=30.4 K 2024-11-21T01:44:40,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/81315e934a6b42c1818d40bed066090a as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/81315e934a6b42c1818d40bed066090a 2024-11-21T01:44:40,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/81315e934a6b42c1818d40bed066090a, entries=150, sequenceid=240, filesize=11.9 K 2024-11-21T01:44:40,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/cbc60e7b87494fc0bc4f38595c5562d9 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/cbc60e7b87494fc0bc4f38595c5562d9 2024-11-21T01:44:40,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/cbc60e7b87494fc0bc4f38595c5562d9, entries=150, sequenceid=240, filesize=11.9 K 2024-11-21T01:44:40,599 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~80.58 KB/82512, currentSize=167.72 KB/171750 for 5835087c277e0e7bdbee836efeecc2f0 in 633ms, sequenceid=240, compaction requested=true 2024-11-21T01:44:40,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:40,599 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:44:40,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5835087c277e0e7bdbee836efeecc2f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:44:40,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:40,601 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:44:40,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5835087c277e0e7bdbee836efeecc2f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:44:40,602 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 116084 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:44:40,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:40,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5835087c277e0e7bdbee836efeecc2f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:44:40,602 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 5835087c277e0e7bdbee836efeecc2f0/A is initiating minor compaction (all files) 2024-11-21T01:44:40,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:40,602 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5835087c277e0e7bdbee836efeecc2f0/A in TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:40,602 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/bc6b7ba3300a488da3ba1b9ab5e8c88d, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/332657bf6e8d473dae18570fe91864c4, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/6a99845586d34d51b99a332aa8c78e10, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/89accf3e359d4f788f11bf2a357dd151] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp, totalSize=113.4 K 2024-11-21T01:44:40,602 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:40,602 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. files: [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/bc6b7ba3300a488da3ba1b9ab5e8c88d, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/332657bf6e8d473dae18570fe91864c4, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/6a99845586d34d51b99a332aa8c78e10, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/89accf3e359d4f788f11bf2a357dd151] 2024-11-21T01:44:40,602 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc6b7ba3300a488da3ba1b9ab5e8c88d, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732153477210 2024-11-21T01:44:40,603 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 332657bf6e8d473dae18570fe91864c4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732153477328 2024-11-21T01:44:40,603 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a99845586d34d51b99a332aa8c78e10, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732153478530 2024-11-21T01:44:40,604 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46518 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:44:40,604 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 5835087c277e0e7bdbee836efeecc2f0/B is initiating minor compaction (all files) 2024-11-21T01:44:40,604 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5835087c277e0e7bdbee836efeecc2f0/B in TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:40,604 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/1e8a5d259c2b4050a5a3145b054fa602, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/a6fc4714f7ce474b8c82134eba1da198, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/5f07c54247bf4267b1068b0a865497f4, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/81315e934a6b42c1818d40bed066090a] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp, totalSize=45.4 K 2024-11-21T01:44:40,604 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89accf3e359d4f788f11bf2a357dd151, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732153479827 2024-11-21T01:44:40,605 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e8a5d259c2b4050a5a3145b054fa602, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732153477210 2024-11-21T01:44:40,606 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting a6fc4714f7ce474b8c82134eba1da198, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732153477328 2024-11-21T01:44:40,606 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f07c54247bf4267b1068b0a865497f4, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732153478530 2024-11-21T01:44:40,607 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 81315e934a6b42c1818d40bed066090a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732153479827 2024-11-21T01:44:40,622 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:40,636 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112113568347603e478ba67c6aa23e77d74e_5835087c277e0e7bdbee836efeecc2f0 store=[table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:40,638 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5835087c277e0e7bdbee836efeecc2f0#B#compaction#154 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:40,639 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/2e185f7e1dc944c6b3cc889f46d583b5 is 50, key is test_row_0/B:col10/1732153479827/Put/seqid=0 2024-11-21T01:44:40,639 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112113568347603e478ba67c6aa23e77d74e_5835087c277e0e7bdbee836efeecc2f0, store=[table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:40,639 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112113568347603e478ba67c6aa23e77d74e_5835087c277e0e7bdbee836efeecc2f0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:40,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742006_1182 (size=12595) 2024-11-21T01:44:40,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742007_1183 (size=4469) 2024-11-21T01:44:40,718 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5835087c277e0e7bdbee836efeecc2f0#A#compaction#153 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:40,718 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/bddb9bcae4124354966afc36bc8f29f1 is 175, key is test_row_0/A:col10/1732153479827/Put/seqid=0 2024-11-21T01:44:40,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T01:44:40,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:40,738 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 5835087c277e0e7bdbee836efeecc2f0 3/3 column families, dataSize=167.72 KB heapSize=446.25 KB 2024-11-21T01:44:40,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=A 2024-11-21T01:44:40,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:40,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=B 2024-11-21T01:44:40,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:40,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=C 2024-11-21T01:44:40,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:40,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T01:44:40,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742008_1184 (size=31549) 2024-11-21T01:44:40,761 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/bddb9bcae4124354966afc36bc8f29f1 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/bddb9bcae4124354966afc36bc8f29f1 2024-11-21T01:44:40,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214e8a28f769624704a5c5e58758ae4dc6_5835087c277e0e7bdbee836efeecc2f0 is 50, key is test_row_0/A:col10/1732153480117/Put/seqid=0 2024-11-21T01:44:40,770 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5835087c277e0e7bdbee836efeecc2f0/A of 5835087c277e0e7bdbee836efeecc2f0 into bddb9bcae4124354966afc36bc8f29f1(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:40,770 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:40,770 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., storeName=5835087c277e0e7bdbee836efeecc2f0/A, priority=12, startTime=1732153480599; duration=0sec 2024-11-21T01:44:40,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:40,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:40,773 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:40,774 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5835087c277e0e7bdbee836efeecc2f0:A 2024-11-21T01:44:40,774 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:44:40,776 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46518 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:44:40,776 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 5835087c277e0e7bdbee836efeecc2f0/C is initiating minor compaction (all files) 2024-11-21T01:44:40,776 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5835087c277e0e7bdbee836efeecc2f0/C in TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:40,776 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/38869b7de0df4f66ac09aa35dd3dd0dc, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/6405a90cc0cb417280b11a4a3ff2236f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/40650b2bbd2141c9a84c9398d151c210, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/cbc60e7b87494fc0bc4f38595c5562d9] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp, totalSize=45.4 K 2024-11-21T01:44:40,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:40,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153540773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:40,777 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38869b7de0df4f66ac09aa35dd3dd0dc, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732153477210 2024-11-21T01:44:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:40,778 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6405a90cc0cb417280b11a4a3ff2236f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732153477328 2024-11-21T01:44:40,778 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40650b2bbd2141c9a84c9398d151c210, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732153478530 2024-11-21T01:44:40,779 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbc60e7b87494fc0bc4f38595c5562d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732153479827 2024-11-21T01:44:40,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742009_1185 (size=12454) 2024-11-21T01:44:40,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:40,785 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214e8a28f769624704a5c5e58758ae4dc6_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214e8a28f769624704a5c5e58758ae4dc6_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:40,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/6387b83d41a24c01860ad5df5634260e, store: [table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:40,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/6387b83d41a24c01860ad5df5634260e is 175, key is test_row_0/A:col10/1732153480117/Put/seqid=0 2024-11-21T01:44:40,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:40,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153540777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:40,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:40,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:40,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153540777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:40,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:40,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:40,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153540777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:40,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:40,796 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5835087c277e0e7bdbee836efeecc2f0#C#compaction#156 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:40,796 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/49dedf7a6a134c7eae2550584cddaa31 is 50, key is test_row_0/C:col10/1732153479827/Put/seqid=0 2024-11-21T01:44:40,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742011_1187 (size=12595) 2024-11-21T01:44:40,833 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/49dedf7a6a134c7eae2550584cddaa31 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/49dedf7a6a134c7eae2550584cddaa31 2024-11-21T01:44:40,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742010_1186 (size=31255) 2024-11-21T01:44:40,835 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/6387b83d41a24c01860ad5df5634260e 2024-11-21T01:44:40,843 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5835087c277e0e7bdbee836efeecc2f0/C of 5835087c277e0e7bdbee836efeecc2f0 into 49dedf7a6a134c7eae2550584cddaa31(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:40,843 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:40,843 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., storeName=5835087c277e0e7bdbee836efeecc2f0/C, priority=12, startTime=1732153480602; duration=0sec 2024-11-21T01:44:40,843 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:40,843 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5835087c277e0e7bdbee836efeecc2f0:C 2024-11-21T01:44:40,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/402f0e3065964dabb46b5c99c53eab72 is 50, key is test_row_0/B:col10/1732153480117/Put/seqid=0 2024-11-21T01:44:40,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742012_1188 (size=12301) 2024-11-21T01:44:40,877 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/402f0e3065964dabb46b5c99c53eab72 2024-11-21T01:44:40,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:40,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153540883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:40,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:40,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/25e885f22aae442fbefd41e5c0f82255 is 50, key is test_row_0/C:col10/1732153480117/Put/seqid=0 2024-11-21T01:44:40,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:40,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153540902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:40,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:40,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153540903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:40,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:40,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153540903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:40,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:40,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:40,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:40,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:40,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742013_1189 (size=12301) 2024-11-21T01:44:40,946 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/25e885f22aae442fbefd41e5c0f82255 2024-11-21T01:44:40,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/6387b83d41a24c01860ad5df5634260e as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/6387b83d41a24c01860ad5df5634260e 2024-11-21T01:44:40,968 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/6387b83d41a24c01860ad5df5634260e, entries=150, sequenceid=291, filesize=30.5 K 2024-11-21T01:44:40,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/402f0e3065964dabb46b5c99c53eab72 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/402f0e3065964dabb46b5c99c53eab72 2024-11-21T01:44:40,976 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/402f0e3065964dabb46b5c99c53eab72, entries=150, sequenceid=291, filesize=12.0 K 2024-11-21T01:44:40,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/25e885f22aae442fbefd41e5c0f82255 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/25e885f22aae442fbefd41e5c0f82255 2024-11-21T01:44:41,025 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/25e885f22aae442fbefd41e5c0f82255, entries=150, sequenceid=291, filesize=12.0 K 2024-11-21T01:44:41,034 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~446.20 KB/456912, currentSize=26.84 KB/27480 for 5835087c277e0e7bdbee836efeecc2f0 in 294ms, sequenceid=291, compaction requested=false 2024-11-21T01:44:41,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:41,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:41,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-21T01:44:41,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-21T01:44:41,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-21T01:44:41,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 929 msec 2024-11-21T01:44:41,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 939 msec 2024-11-21T01:44:41,114 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/2e185f7e1dc944c6b3cc889f46d583b5 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/2e185f7e1dc944c6b3cc889f46d583b5 2024-11-21T01:44:41,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:41,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5835087c277e0e7bdbee836efeecc2f0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T01:44:41,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=A 2024-11-21T01:44:41,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:41,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=B 2024-11-21T01:44:41,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:41,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=C 2024-11-21T01:44:41,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:41,120 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5835087c277e0e7bdbee836efeecc2f0/B of 5835087c277e0e7bdbee836efeecc2f0 into 2e185f7e1dc944c6b3cc889f46d583b5(size=12.3 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:41,120 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:41,120 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., storeName=5835087c277e0e7bdbee836efeecc2f0/B, priority=12, startTime=1732153480601; duration=0sec 2024-11-21T01:44:41,121 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:41,121 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5835087c277e0e7bdbee836efeecc2f0:B 2024-11-21T01:44:41,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d4d109ba343f4f55ae38f1fbe96e99a0_5835087c277e0e7bdbee836efeecc2f0 is 50, key is test_row_0/A:col10/1732153481107/Put/seqid=0 2024-11-21T01:44:41,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732153481146, max timestamp=1732153481164Num uniques 859; 2024-11-21T01:44:41,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-21T01:44:41,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742014_1190 (size=14994) 2024-11-21T01:44:41,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732153481146, max timestamp=1732153481164Num uniques 859; 2024-11-21T01:44:41,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-21T01:44:41,170 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:41,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732153481146, max timestamp=1732153481164Num uniques 859; 2024-11-21T01:44:41,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-21T01:44:41,185 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d4d109ba343f4f55ae38f1fbe96e99a0_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d4d109ba343f4f55ae38f1fbe96e99a0_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:41,186 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/9880acac652344c28ecd5044f9eb3a08, store: [table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:41,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/9880acac652344c28ecd5044f9eb3a08 is 175, key is test_row_0/A:col10/1732153481107/Put/seqid=0 2024-11-21T01:44:41,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742015_1191 (size=39949) 2024-11-21T01:44:41,202 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=303, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/9880acac652344c28ecd5044f9eb3a08 2024-11-21T01:44:41,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:41,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153541208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:41,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/b9321bbf2a724fdd9b98d13adc0b32bd is 50, key is test_row_0/B:col10/1732153481107/Put/seqid=0 2024-11-21T01:44:41,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:41,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:41,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:41,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:41,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153541210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:41,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:41,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153541211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:41,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:41,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:41,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:41,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153541213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:41,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:41,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:41,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:41,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T01:44:41,253 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:44:41,254 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:44:41,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-21T01:44:41,256 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:44:41,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T01:44:41,257 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:44:41,257 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:44:41,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742016_1192 (size=12301) 2024-11-21T01:44:41,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/b9321bbf2a724fdd9b98d13adc0b32bd 2024-11-21T01:44:41,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/832655905ec443458dafddfcb3886734 is 50, key is test_row_0/C:col10/1732153481107/Put/seqid=0 2024-11-21T01:44:41,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:41,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153541327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:41,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:41,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:41,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:41,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742017_1193 (size=12301) 2024-11-21T01:44:41,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153541333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:41,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:41,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:41,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:41,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153541333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:41,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:41,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153541333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:41,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:41,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:41,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:41,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:41,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:41,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T01:44:41,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-21T01:44:41,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:41,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:41,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:41,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:41,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:41,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:41,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:41,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153541543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:41,555 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:41,555 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:41,555 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:41,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:41,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153541563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:41,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:41,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:41,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:41,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-21T01:44:41,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:41,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153541562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:41,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:41,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:41,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:41,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:41,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:41,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:41,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:41,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:41,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T01:44:41,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:41,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:41,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153541577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:41,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-21T01:44:41,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:41,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:41,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:41,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:41,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:41,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:41,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/832655905ec443458dafddfcb3886734 2024-11-21T01:44:41,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/9880acac652344c28ecd5044f9eb3a08 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/9880acac652344c28ecd5044f9eb3a08 2024-11-21T01:44:41,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/9880acac652344c28ecd5044f9eb3a08, entries=200, sequenceid=303, filesize=39.0 K 2024-11-21T01:44:41,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/b9321bbf2a724fdd9b98d13adc0b32bd as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/b9321bbf2a724fdd9b98d13adc0b32bd 2024-11-21T01:44:41,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/b9321bbf2a724fdd9b98d13adc0b32bd, entries=150, sequenceid=303, filesize=12.0 K 2024-11-21T01:44:41,776 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-21T01:44:41,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/832655905ec443458dafddfcb3886734 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/832655905ec443458dafddfcb3886734 2024-11-21T01:44:41,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/832655905ec443458dafddfcb3886734, entries=150, sequenceid=303, filesize=12.0 K 2024-11-21T01:44:41,788 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 5835087c277e0e7bdbee836efeecc2f0 in 670ms, sequenceid=303, compaction requested=true 2024-11-21T01:44:41,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:41,788 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:41,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5835087c277e0e7bdbee836efeecc2f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:44:41,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:41,788 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:41,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5835087c277e0e7bdbee836efeecc2f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:44:41,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:41,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5835087c277e0e7bdbee836efeecc2f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:44:41,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:41,789 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102753 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:41,789 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 5835087c277e0e7bdbee836efeecc2f0/A is initiating minor compaction (all files) 2024-11-21T01:44:41,790 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5835087c277e0e7bdbee836efeecc2f0/A in TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:41,790 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/bddb9bcae4124354966afc36bc8f29f1, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/6387b83d41a24c01860ad5df5634260e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/9880acac652344c28ecd5044f9eb3a08] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp, totalSize=100.3 K 2024-11-21T01:44:41,790 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:41,790 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. files: [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/bddb9bcae4124354966afc36bc8f29f1, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/6387b83d41a24c01860ad5df5634260e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/9880acac652344c28ecd5044f9eb3a08] 2024-11-21T01:44:41,790 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting bddb9bcae4124354966afc36bc8f29f1, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732153479827 2024-11-21T01:44:41,791 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:41,791 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 5835087c277e0e7bdbee836efeecc2f0/B is initiating minor compaction (all files) 2024-11-21T01:44:41,791 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6387b83d41a24c01860ad5df5634260e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732153480097 2024-11-21T01:44:41,791 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5835087c277e0e7bdbee836efeecc2f0/B in TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:41,791 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/2e185f7e1dc944c6b3cc889f46d583b5, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/402f0e3065964dabb46b5c99c53eab72, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/b9321bbf2a724fdd9b98d13adc0b32bd] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp, totalSize=36.3 K 2024-11-21T01:44:41,791 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9880acac652344c28ecd5044f9eb3a08, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732153480775 2024-11-21T01:44:41,791 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e185f7e1dc944c6b3cc889f46d583b5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732153479827 2024-11-21T01:44:41,792 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 402f0e3065964dabb46b5c99c53eab72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732153480097 2024-11-21T01:44:41,793 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting b9321bbf2a724fdd9b98d13adc0b32bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732153480775 2024-11-21T01:44:41,802 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:41,810 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411217e70a17b329641f9b6241722f626c132_5835087c277e0e7bdbee836efeecc2f0 store=[table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:41,812 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411217e70a17b329641f9b6241722f626c132_5835087c277e0e7bdbee836efeecc2f0, store=[table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:41,812 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411217e70a17b329641f9b6241722f626c132_5835087c277e0e7bdbee836efeecc2f0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:41,819 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5835087c277e0e7bdbee836efeecc2f0#B#compaction#163 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:41,820 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/fe32b45b616d4ed39bd13e95d8a791ea is 50, key is test_row_0/B:col10/1732153481107/Put/seqid=0 2024-11-21T01:44:41,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742018_1194 (size=4469) 2024-11-21T01:44:41,851 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5835087c277e0e7bdbee836efeecc2f0#A#compaction#162 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:41,851 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/d291762108ab48a29abf8f0fbadf66f1 is 175, key is test_row_0/A:col10/1732153481107/Put/seqid=0 2024-11-21T01:44:41,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:41,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5835087c277e0e7bdbee836efeecc2f0 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-21T01:44:41,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=A 2024-11-21T01:44:41,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:41,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=B 2024-11-21T01:44:41,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:41,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=C 2024-11-21T01:44:41,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:41,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T01:44:41,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742019_1195 (size=12847) 2024-11-21T01:44:41,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-21T01:44:41,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:41,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:41,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:41,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:41,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:41,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:41,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:41,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153541899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:41,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:41,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:41,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:41,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:41,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:41,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153541901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:41,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153541904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:41,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:41,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153541906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:41,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:41,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:41,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:41,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:41,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:41,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:41,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:41,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:41,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:41,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742020_1196 (size=31801) 2024-11-21T01:44:41,938 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/d291762108ab48a29abf8f0fbadf66f1 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/d291762108ab48a29abf8f0fbadf66f1 2024-11-21T01:44:41,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112197a93d369cc346fd9434d7e5ffd88f21_5835087c277e0e7bdbee836efeecc2f0 is 50, key is test_row_0/A:col10/1732153481866/Put/seqid=0 2024-11-21T01:44:41,945 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5835087c277e0e7bdbee836efeecc2f0/A of 5835087c277e0e7bdbee836efeecc2f0 into d291762108ab48a29abf8f0fbadf66f1(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:41,946 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:41,946 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., storeName=5835087c277e0e7bdbee836efeecc2f0/A, priority=13, startTime=1732153481788; duration=0sec 2024-11-21T01:44:41,953 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:41,953 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5835087c277e0e7bdbee836efeecc2f0:A 2024-11-21T01:44:41,953 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:41,959 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:41,959 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 5835087c277e0e7bdbee836efeecc2f0/C is initiating minor compaction (all files) 2024-11-21T01:44:41,959 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5835087c277e0e7bdbee836efeecc2f0/C in TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:41,959 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/49dedf7a6a134c7eae2550584cddaa31, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/25e885f22aae442fbefd41e5c0f82255, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/832655905ec443458dafddfcb3886734] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp, totalSize=36.3 K 2024-11-21T01:44:41,963 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49dedf7a6a134c7eae2550584cddaa31, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732153479827 2024-11-21T01:44:41,964 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25e885f22aae442fbefd41e5c0f82255, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732153480097 2024-11-21T01:44:41,967 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 832655905ec443458dafddfcb3886734, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732153480775 2024-11-21T01:44:41,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742021_1197 (size=14994) 2024-11-21T01:44:41,986 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,015 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112197a93d369cc346fd9434d7e5ffd88f21_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112197a93d369cc346fd9434d7e5ffd88f21_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:42,018 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5835087c277e0e7bdbee836efeecc2f0#C#compaction#165 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:42,019 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/4b7129d62ee043959ecfb058ef2626eb is 50, key is test_row_0/C:col10/1732153481107/Put/seqid=0 2024-11-21T01:44:42,028 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/80f863c56cb34b3ebfee703776532e32, store: [table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:42,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/80f863c56cb34b3ebfee703776532e32 is 175, key is test_row_0/A:col10/1732153481866/Put/seqid=0 2024-11-21T01:44:42,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:42,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153542023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:42,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:42,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:42,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:42,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:42,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153542023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:42,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:42,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:42,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:42,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:42,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153542023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:42,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:42,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:42,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:42,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:42,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153542024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:42,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:42,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:42,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:42,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-21T01:44:42,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:42,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:42,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:42,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:42,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:42,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:42,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742022_1198 (size=12847) 2024-11-21T01:44:42,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742023_1199 (size=39949) 2024-11-21T01:44:42,091 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=340, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/80f863c56cb34b3ebfee703776532e32 2024-11-21T01:44:42,098 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/4b7129d62ee043959ecfb058ef2626eb as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/4b7129d62ee043959ecfb058ef2626eb 2024-11-21T01:44:42,112 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5835087c277e0e7bdbee836efeecc2f0/C of 5835087c277e0e7bdbee836efeecc2f0 into 4b7129d62ee043959ecfb058ef2626eb(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:42,112 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:42,112 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., storeName=5835087c277e0e7bdbee836efeecc2f0/C, priority=13, startTime=1732153481789; duration=0sec 2024-11-21T01:44:42,112 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:42,112 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5835087c277e0e7bdbee836efeecc2f0:C 2024-11-21T01:44:42,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/d7acf32b9c3745aa80c9895e1cff4ae8 is 50, key is test_row_0/B:col10/1732153481866/Put/seqid=0 2024-11-21T01:44:42,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742024_1200 (size=12301) 2024-11-21T01:44:42,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/d7acf32b9c3745aa80c9895e1cff4ae8 2024-11-21T01:44:42,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/5b661603cb0b473d98d599317856b027 is 50, key is test_row_0/C:col10/1732153481866/Put/seqid=0 2024-11-21T01:44:42,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742025_1201 (size=12301) 2024-11-21T01:44:42,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/5b661603cb0b473d98d599317856b027 2024-11-21T01:44:42,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/80f863c56cb34b3ebfee703776532e32 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/80f863c56cb34b3ebfee703776532e32 2024-11-21T01:44:42,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/80f863c56cb34b3ebfee703776532e32, entries=200, sequenceid=340, filesize=39.0 K 2024-11-21T01:44:42,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/d7acf32b9c3745aa80c9895e1cff4ae8 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/d7acf32b9c3745aa80c9895e1cff4ae8 2024-11-21T01:44:42,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/d7acf32b9c3745aa80c9895e1cff4ae8, entries=150, sequenceid=340, filesize=12.0 K 2024-11-21T01:44:42,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/5b661603cb0b473d98d599317856b027 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/5b661603cb0b473d98d599317856b027 2024-11-21T01:44:42,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/5b661603cb0b473d98d599317856b027, entries=150, sequenceid=340, filesize=12.0 K 2024-11-21T01:44:42,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,183 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for 5835087c277e0e7bdbee836efeecc2f0 in 312ms, sequenceid=340, compaction requested=false 2024-11-21T01:44:42,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:42,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-21T01:44:42,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:42,207 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 5835087c277e0e7bdbee836efeecc2f0 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-21T01:44:42,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=A 2024-11-21T01:44:42,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:42,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=B 2024-11-21T01:44:42,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:42,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=C 2024-11-21T01:44:42,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:42,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121413cb3d681f84c92a976634f41e7eb70_5835087c277e0e7bdbee836efeecc2f0 is 50, key is test_row_1/A:col10/1732153481902/Put/seqid=0 2024-11-21T01:44:42,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742026_1202 (size=9914) 2024-11-21T01:44:42,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:42,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:42,298 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/fe32b45b616d4ed39bd13e95d8a791ea as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/fe32b45b616d4ed39bd13e95d8a791ea 2024-11-21T01:44:42,316 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5835087c277e0e7bdbee836efeecc2f0/B of 5835087c277e0e7bdbee836efeecc2f0 into fe32b45b616d4ed39bd13e95d8a791ea(size=12.5 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:42,316 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:42,316 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., storeName=5835087c277e0e7bdbee836efeecc2f0/B, priority=13, startTime=1732153481788; duration=0sec 2024-11-21T01:44:42,316 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:42,316 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5835087c277e0e7bdbee836efeecc2f0:B 2024-11-21T01:44:42,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732153482293, max timestamp=1732153482327Num uniques 859; 2024-11-21T01:44:42,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-21T01:44:42,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732153482293, max timestamp=1732153482327Num uniques 859; 2024-11-21T01:44:42,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-21T01:44:42,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732153482293, max timestamp=1732153482327Num uniques 859; 2024-11-21T01:44:42,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-21T01:44:42,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:42,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153542376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:42,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:42,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:42,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:42,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:42,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153542378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:42,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:42,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:42,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:42,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:42,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153542379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:42,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:42,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:42,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153542379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:42,387 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:42,387 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:42,387 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:42,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T01:44:42,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:42,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153542483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:42,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:42,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:42,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:42,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153542492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:42,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153542493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:42,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:42,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:42,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:42,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153542493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:42,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:42,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:42,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:42,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:42,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:42,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:42,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:42,629 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121413cb3d681f84c92a976634f41e7eb70_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121413cb3d681f84c92a976634f41e7eb70_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:42,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/cad19ebcdfbe45a9a4aacbaeb4d15771, store: [table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:42,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/cad19ebcdfbe45a9a4aacbaeb4d15771 is 175, key is test_row_1/A:col10/1732153481902/Put/seqid=0 2024-11-21T01:44:42,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742027_1203 (size=22561) 2024-11-21T01:44:42,651 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=351, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/cad19ebcdfbe45a9a4aacbaeb4d15771 2024-11-21T01:44:42,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/80c2bd3bf85d4d299d301a1d88e7ea91 is 50, key is test_row_1/B:col10/1732153481902/Put/seqid=0 2024-11-21T01:44:42,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742028_1204 (size=9857) 2024-11-21T01:44:42,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:42,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153542703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:42,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:42,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:42,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:42,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153542703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:42,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:42,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:42,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:42,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:42,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:42,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153542703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:42,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:42,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:42,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:42,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:42,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153542703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:42,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:42,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:42,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:43,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:43,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153543012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:43,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:43,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:43,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:43,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153543024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:43,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:43,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:43,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:43,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153543024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:43,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:43,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:43,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:43,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:43,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:43,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153543024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:43,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:43,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:43,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:43,080 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/80c2bd3bf85d4d299d301a1d88e7ea91 2024-11-21T01:44:43,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/04464739c90348e7bdd5a62bf9c94e0b is 50, key is test_row_1/C:col10/1732153481902/Put/seqid=0 2024-11-21T01:44:43,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742029_1205 (size=9857) 2024-11-21T01:44:43,118 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/04464739c90348e7bdd5a62bf9c94e0b 2024-11-21T01:44:43,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/cad19ebcdfbe45a9a4aacbaeb4d15771 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/cad19ebcdfbe45a9a4aacbaeb4d15771 2024-11-21T01:44:43,139 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/cad19ebcdfbe45a9a4aacbaeb4d15771, entries=100, sequenceid=351, filesize=22.0 K 2024-11-21T01:44:43,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/80c2bd3bf85d4d299d301a1d88e7ea91 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/80c2bd3bf85d4d299d301a1d88e7ea91 2024-11-21T01:44:43,154 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/80c2bd3bf85d4d299d301a1d88e7ea91, entries=100, sequenceid=351, filesize=9.6 K 2024-11-21T01:44:43,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/04464739c90348e7bdd5a62bf9c94e0b as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/04464739c90348e7bdd5a62bf9c94e0b 2024-11-21T01:44:43,164 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/04464739c90348e7bdd5a62bf9c94e0b, entries=100, sequenceid=351, filesize=9.6 K 2024-11-21T01:44:43,165 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=154.31 KB/158010 for 5835087c277e0e7bdbee836efeecc2f0 in 958ms, sequenceid=351, compaction requested=true 2024-11-21T01:44:43,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:43,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:43,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-21T01:44:43,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-21T01:44:43,173 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-21T01:44:43,173 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9140 sec 2024-11-21T01:44:43,176 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.9200 sec 2024-11-21T01:44:43,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T01:44:43,404 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:44:43,404 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:44:43,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-21T01:44:43,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T01:44:43,408 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:44:43,408 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:44:43,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:44:43,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T01:44:43,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732153482366, max timestamp=1732153483541Num uniques 1218; 2024-11-21T01:44:43,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732153482366, max timestamp=1732153483541Num uniques 1218; 2024-11-21T01:44:43,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=55778, new segment data size=9978, suffix heap size=154640,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1218, new segment cells count=218 2024-11-21T01:44:43,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=55778, new segment data size=9978, suffix heap size=154640,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1218, new segment cells count=218 2024-11-21T01:44:43,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5835087c277e0e7bdbee836efeecc2f0 3/3 column families, dataSize=78.27 KB heapSize=209.27 KB 2024-11-21T01:44:43,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=A 2024-11-21T01:44:43,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:43,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=B 2024-11-21T01:44:43,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:43,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=C 2024-11-21T01:44:43,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T01:44:43,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:43,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-21T01:44:43,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:43,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:43,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:43,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:43,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:43,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:43,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e4c1ed92dddb40ea9d37e31b0ce57c42_5835087c277e0e7bdbee836efeecc2f0 is 50, key is test_row_0/A:col10/1732153483539/Put/seqid=0 2024-11-21T01:44:43,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742030_1206 (size=12454) 2024-11-21T01:44:43,582 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:43,587 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e4c1ed92dddb40ea9d37e31b0ce57c42_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e4c1ed92dddb40ea9d37e31b0ce57c42_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:43,589 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/440870c40e4e4c0285d42f913e567bc5, store: [table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:43,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/440870c40e4e4c0285d42f913e567bc5 is 175, key is test_row_0/A:col10/1732153483539/Put/seqid=0 2024-11-21T01:44:43,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:43,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153543593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:43,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742031_1207 (size=31255) 2024-11-21T01:44:43,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:43,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:43,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:43,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:43,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153543596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:43,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:43,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153543597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:43,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:43,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:43,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:43,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:43,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:43,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:43,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:43,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153543610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:43,619 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=390, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/440870c40e4e4c0285d42f913e567bc5 2024-11-21T01:44:43,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/07830ee979e1464384c5f150f4d6e848 is 50, key is test_row_0/B:col10/1732153483539/Put/seqid=0 2024-11-21T01:44:43,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742032_1208 (size=12301) 2024-11-21T01:44:43,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153543713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:43,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153543713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:43,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:43,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:43,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153543713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:43,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:43,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:43,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:43,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:43,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:43,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:43,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:43,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-21T01:44:43,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:43,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:43,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:43,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:43,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T01:44:43,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153543723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:43,727 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:43,727 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:43,727 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:43,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-21T01:44:43,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:43,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:43,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:43,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:43,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:43,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:43,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153543923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:43,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:43,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:43,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:43,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153543923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:43,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:43,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:43,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153543923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:43,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:43,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:43,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:43,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:43,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153543933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:43,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:43,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:43,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:44,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-21T01:44:44,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:44,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:44,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:44,028 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:44,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:44,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:44,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T01:44:44,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/07830ee979e1464384c5f150f4d6e848 2024-11-21T01:44:44,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/33d5eac8aabb4e6c824789534e830f74 is 50, key is test_row_0/C:col10/1732153483539/Put/seqid=0 2024-11-21T01:44:44,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742033_1209 (size=12301) 2024-11-21T01:44:44,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-21T01:44:44,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:44,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:44,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:44,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:44,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:44,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:44,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:44,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153544233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:44,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:44,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153544233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:44,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:44,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153544233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:44,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:44,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:44,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:44,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:44,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:44,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:44,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:44,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:44,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:44,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:44,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153544242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:44,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:44,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:44,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:44,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-21T01:44:44,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:44,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:44,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:44,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:44,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:44,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:44,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/33d5eac8aabb4e6c824789534e830f74 2024-11-21T01:44:44,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-21T01:44:44,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:44,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:44,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:44,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:44,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:44,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:44,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/440870c40e4e4c0285d42f913e567bc5 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/440870c40e4e4c0285d42f913e567bc5 2024-11-21T01:44:44,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/440870c40e4e4c0285d42f913e567bc5, entries=150, sequenceid=390, filesize=30.5 K 2024-11-21T01:44:44,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/07830ee979e1464384c5f150f4d6e848 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/07830ee979e1464384c5f150f4d6e848 2024-11-21T01:44:44,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/07830ee979e1464384c5f150f4d6e848, entries=150, sequenceid=390, filesize=12.0 K 2024-11-21T01:44:44,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/33d5eac8aabb4e6c824789534e830f74 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/33d5eac8aabb4e6c824789534e830f74 2024-11-21T01:44:44,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/33d5eac8aabb4e6c824789534e830f74, entries=150, sequenceid=390, filesize=12.0 K 2024-11-21T01:44:44,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~78.27 KB/80150, heapSize ~209.22 KB/214240, currentSize=114.05 KB/116790 for 5835087c277e0e7bdbee836efeecc2f0 in 981ms, sequenceid=390, compaction requested=true 2024-11-21T01:44:44,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:44,532 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:44:44,534 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125566 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:44:44,534 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 5835087c277e0e7bdbee836efeecc2f0/A is initiating minor compaction (all files) 2024-11-21T01:44:44,534 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5835087c277e0e7bdbee836efeecc2f0/A in TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:44,534 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/d291762108ab48a29abf8f0fbadf66f1, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/80f863c56cb34b3ebfee703776532e32, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/cad19ebcdfbe45a9a4aacbaeb4d15771, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/440870c40e4e4c0285d42f913e567bc5] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp, totalSize=122.6 K 2024-11-21T01:44:44,534 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:44,534 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. files: [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/d291762108ab48a29abf8f0fbadf66f1, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/80f863c56cb34b3ebfee703776532e32, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/cad19ebcdfbe45a9a4aacbaeb4d15771, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/440870c40e4e4c0285d42f913e567bc5] 2024-11-21T01:44:44,534 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting d291762108ab48a29abf8f0fbadf66f1, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732153480775 2024-11-21T01:44:44,535 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80f863c56cb34b3ebfee703776532e32, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732153481198 2024-11-21T01:44:44,535 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting cad19ebcdfbe45a9a4aacbaeb4d15771, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732153481899 2024-11-21T01:44:44,537 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 440870c40e4e4c0285d42f913e567bc5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732153482377 2024-11-21T01:44:44,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T01:44:44,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5835087c277e0e7bdbee836efeecc2f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:44:44,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:44,547 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:44:44,548 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47306 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:44:44,549 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 5835087c277e0e7bdbee836efeecc2f0/B is initiating minor compaction (all files) 2024-11-21T01:44:44,549 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5835087c277e0e7bdbee836efeecc2f0/B in TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:44,549 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/fe32b45b616d4ed39bd13e95d8a791ea, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/d7acf32b9c3745aa80c9895e1cff4ae8, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/80c2bd3bf85d4d299d301a1d88e7ea91, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/07830ee979e1464384c5f150f4d6e848] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp, totalSize=46.2 K 2024-11-21T01:44:44,549 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting fe32b45b616d4ed39bd13e95d8a791ea, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732153480775 2024-11-21T01:44:44,550 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting d7acf32b9c3745aa80c9895e1cff4ae8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732153481209 2024-11-21T01:44:44,550 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 80c2bd3bf85d4d299d301a1d88e7ea91, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732153481899 2024-11-21T01:44:44,550 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 07830ee979e1464384c5f150f4d6e848, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732153482377 2024-11-21T01:44:44,556 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:44,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5835087c277e0e7bdbee836efeecc2f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:44:44,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:44,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5835087c277e0e7bdbee836efeecc2f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:44:44,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:44,568 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121e3c090e5c25d4b29a2085d64d74e8a15_5835087c277e0e7bdbee836efeecc2f0 store=[table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:44,571 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121e3c090e5c25d4b29a2085d64d74e8a15_5835087c277e0e7bdbee836efeecc2f0, store=[table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:44,571 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e3c090e5c25d4b29a2085d64d74e8a15_5835087c277e0e7bdbee836efeecc2f0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:44,575 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5835087c277e0e7bdbee836efeecc2f0#B#compaction#175 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:44,576 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/d87d433f45124c2599db820c54e70073 is 50, key is test_row_0/B:col10/1732153483539/Put/seqid=0 2024-11-21T01:44:44,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742034_1210 (size=4469) 2024-11-21T01:44:44,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-21T01:44:44,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:44,649 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 5835087c277e0e7bdbee836efeecc2f0 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T01:44:44,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=A 2024-11-21T01:44:44,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:44,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=B 2024-11-21T01:44:44,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:44,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=C 2024-11-21T01:44:44,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:44,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112138e14a9042c944a4b50643181dec3c49_5835087c277e0e7bdbee836efeecc2f0 is 50, key is test_row_0/A:col10/1732153483596/Put/seqid=0 2024-11-21T01:44:44,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742035_1211 (size=12983) 2024-11-21T01:44:44,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742036_1212 (size=12454) 2024-11-21T01:44:44,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:44,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:44,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:44,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153544766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:44,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:44,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:44,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:44,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:44,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153544768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:44,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:44,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:44,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:44,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:44,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153544770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:44,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:44,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:44,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:44,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:44,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153544772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:44,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:44,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:44,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:44,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:44,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153544883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:44,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:44,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153544883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:44,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:44,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:44,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:44,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:44,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:44,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:44,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:44,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153544883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:44,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:44,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153544883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:44,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:44,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:44,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:44,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:44,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:44,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:45,045 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5835087c277e0e7bdbee836efeecc2f0#A#compaction#174 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:45,046 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/438aa49f78054b138d5c0e67a11938c0 is 175, key is test_row_0/A:col10/1732153483539/Put/seqid=0 2024-11-21T01:44:45,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742037_1213 (size=31937) 2024-11-21T01:44:45,075 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/d87d433f45124c2599db820c54e70073 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/d87d433f45124c2599db820c54e70073 2024-11-21T01:44:45,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:45,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:45,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153545093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:45,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:45,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153545093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:45,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:45,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153545094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:45,095 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112138e14a9042c944a4b50643181dec3c49_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112138e14a9042c944a4b50643181dec3c49_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:45,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:45,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:45,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:45,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:45,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153545094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:45,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:45,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:45,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:45,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:45,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:45,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:45,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:45,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:45,096 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5835087c277e0e7bdbee836efeecc2f0/B of 5835087c277e0e7bdbee836efeecc2f0 into d87d433f45124c2599db820c54e70073(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:45,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:45,096 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:45,096 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., storeName=5835087c277e0e7bdbee836efeecc2f0/B, priority=12, startTime=1732153484547; duration=0sec 2024-11-21T01:44:45,096 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:45,096 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5835087c277e0e7bdbee836efeecc2f0:B 2024-11-21T01:44:45,096 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:44:45,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/2a117c6f6ddc4fe4aa2b2db1b758eda8, store: [table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:45,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/2a117c6f6ddc4fe4aa2b2db1b758eda8 is 175, key is test_row_0/A:col10/1732153483596/Put/seqid=0 2024-11-21T01:44:45,101 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47306 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:44:45,102 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 5835087c277e0e7bdbee836efeecc2f0/C is initiating minor compaction (all files) 2024-11-21T01:44:45,102 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5835087c277e0e7bdbee836efeecc2f0/C in TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:45,102 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/4b7129d62ee043959ecfb058ef2626eb, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/5b661603cb0b473d98d599317856b027, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/04464739c90348e7bdd5a62bf9c94e0b, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/33d5eac8aabb4e6c824789534e830f74] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp, totalSize=46.2 K 2024-11-21T01:44:45,103 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b7129d62ee043959ecfb058ef2626eb, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732153480775 2024-11-21T01:44:45,104 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b661603cb0b473d98d599317856b027, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732153481209 2024-11-21T01:44:45,104 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 04464739c90348e7bdd5a62bf9c94e0b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732153481899 2024-11-21T01:44:45,105 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 33d5eac8aabb4e6c824789534e830f74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732153482377 2024-11-21T01:44:45,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742038_1214 (size=31255) 2024-11-21T01:44:45,127 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5835087c277e0e7bdbee836efeecc2f0#C#compaction#177 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:45,128 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/d92c75c602b24413a1bcf556b8e92fb3 is 50, key is test_row_0/C:col10/1732153483539/Put/seqid=0 2024-11-21T01:44:45,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742039_1215 (size=12983) 2024-11-21T01:44:45,150 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/d92c75c602b24413a1bcf556b8e92fb3 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/d92c75c602b24413a1bcf556b8e92fb3 2024-11-21T01:44:45,156 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5835087c277e0e7bdbee836efeecc2f0/C of 5835087c277e0e7bdbee836efeecc2f0 into d92c75c602b24413a1bcf556b8e92fb3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:45,156 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:45,156 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., storeName=5835087c277e0e7bdbee836efeecc2f0/C, priority=12, startTime=1732153484563; duration=0sec 2024-11-21T01:44:45,156 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:45,156 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5835087c277e0e7bdbee836efeecc2f0:C 2024-11-21T01:44:45,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153545406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:45,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153545406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:45,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:45,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:45,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:45,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153545406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:45,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:45,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:45,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:45,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:45,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:45,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:45,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:45,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153545406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:45,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:45,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:45,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:45,467 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/438aa49f78054b138d5c0e67a11938c0 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/438aa49f78054b138d5c0e67a11938c0 2024-11-21T01:44:45,481 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5835087c277e0e7bdbee836efeecc2f0/A of 5835087c277e0e7bdbee836efeecc2f0 into 438aa49f78054b138d5c0e67a11938c0(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:45,481 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:45,481 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., storeName=5835087c277e0e7bdbee836efeecc2f0/A, priority=12, startTime=1732153484532; duration=0sec 2024-11-21T01:44:45,482 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:45,482 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5835087c277e0e7bdbee836efeecc2f0:A 2024-11-21T01:44:45,519 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=410, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/2a117c6f6ddc4fe4aa2b2db1b758eda8 2024-11-21T01:44:45,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/6bf8f8ac0b5943b1ad6639101b7f490b is 50, key is test_row_0/B:col10/1732153483596/Put/seqid=0 2024-11-21T01:44:45,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T01:44:45,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742040_1216 (size=12301) 2024-11-21T01:44:45,570 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/6bf8f8ac0b5943b1ad6639101b7f490b 2024-11-21T01:44:45,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/82eba9bc6e4f42ae9e69ca5e5dbce535 is 50, key is test_row_0/C:col10/1732153483596/Put/seqid=0 2024-11-21T01:44:45,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742041_1217 (size=12301) 2024-11-21T01:44:45,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153545913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:45,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:45,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:45,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:45,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:45,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153545922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:45,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:45,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:45,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:45,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:45,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153545923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:45,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:45,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:45,931 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:45,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:45,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153545923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:45,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:45,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:45,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:45,999 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/82eba9bc6e4f42ae9e69ca5e5dbce535 2024-11-21T01:44:46,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/2a117c6f6ddc4fe4aa2b2db1b758eda8 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/2a117c6f6ddc4fe4aa2b2db1b758eda8 2024-11-21T01:44:46,013 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/2a117c6f6ddc4fe4aa2b2db1b758eda8, entries=150, sequenceid=410, filesize=30.5 K 2024-11-21T01:44:46,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/6bf8f8ac0b5943b1ad6639101b7f490b as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/6bf8f8ac0b5943b1ad6639101b7f490b 2024-11-21T01:44:46,027 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/6bf8f8ac0b5943b1ad6639101b7f490b, entries=150, sequenceid=410, filesize=12.0 K 2024-11-21T01:44:46,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/82eba9bc6e4f42ae9e69ca5e5dbce535 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/82eba9bc6e4f42ae9e69ca5e5dbce535 2024-11-21T01:44:46,042 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/82eba9bc6e4f42ae9e69ca5e5dbce535, entries=150, sequenceid=410, filesize=12.0 K 2024-11-21T01:44:46,045 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 5835087c277e0e7bdbee836efeecc2f0 in 1396ms, sequenceid=410, compaction requested=false 2024-11-21T01:44:46,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:46,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:46,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-21T01:44:46,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-21T01:44:46,049 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-21T01:44:46,049 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6390 sec 2024-11-21T01:44:46,051 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 2.6450 sec 2024-11-21T01:44:46,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:46,929 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5835087c277e0e7bdbee836efeecc2f0 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T01:44:46,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=A 2024-11-21T01:44:46,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:46,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=B 2024-11-21T01:44:46,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:46,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=C 2024-11-21T01:44:46,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:46,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e2107c0268fe40a7b446369c63ed8ce3_5835087c277e0e7bdbee836efeecc2f0 is 50, key is test_row_0/A:col10/1732153484747/Put/seqid=0 2024-11-21T01:44:46,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742042_1218 (size=14994) 2024-11-21T01:44:46,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:46,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153546980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:46,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:46,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153546980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:46,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:46,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:46,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:46,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:46,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:46,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153546988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:46,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:46,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:46,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:46,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153546989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:46,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:46,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:46,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:46,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:46,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:46,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:47,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:47,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153547093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:47,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:47,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:47,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:47,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:47,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153547093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:47,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:47,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:47,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:47,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:47,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153547093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:47,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:47,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:47,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:47,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:47,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153547093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:47,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:47,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:47,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:47,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:47,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153547303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:47,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:47,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:47,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:47,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:47,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153547303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:47,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:47,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:47,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153547303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:47,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:47,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:47,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:47,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:47,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:47,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153547303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:47,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:47,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:47,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:47,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:47,382 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:47,390 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e2107c0268fe40a7b446369c63ed8ce3_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e2107c0268fe40a7b446369c63ed8ce3_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:47,391 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/cd6aeefc19c64d51a9d533dce0342b60, store: [table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:47,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/cd6aeefc19c64d51a9d533dce0342b60 is 175, key is test_row_0/A:col10/1732153484747/Put/seqid=0 2024-11-21T01:44:47,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742043_1219 (size=39949) 2024-11-21T01:44:47,399 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=426, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/cd6aeefc19c64d51a9d533dce0342b60 2024-11-21T01:44:47,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/eb53cf430e1246f8858058fc4f0dd9e1 is 50, key is test_row_0/B:col10/1732153484747/Put/seqid=0 2024-11-21T01:44:47,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742044_1220 (size=12301) 2024-11-21T01:44:47,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T01:44:47,583 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:44:47,584 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:44:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-21T01:44:47,587 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:44:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-21T01:44:47,594 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:44:47,594 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:44:47,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153547612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:47,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153547612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:47,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:47,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:47,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:47,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:47,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:47,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:47,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153547613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:47,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153547613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:47,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:47,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:47,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-21T01:44:47,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-21T01:44:47,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:47,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:47,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:47,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:47,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:47,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/eb53cf430e1246f8858058fc4f0dd9e1 2024-11-21T01:44:47,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/00043a8198d84f7a93fcc0516d3c20fe is 50, key is test_row_0/C:col10/1732153484747/Put/seqid=0 2024-11-21T01:44:47,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742045_1221 (size=12301) 2024-11-21T01:44:47,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/00043a8198d84f7a93fcc0516d3c20fe 2024-11-21T01:44:47,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/cd6aeefc19c64d51a9d533dce0342b60 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/cd6aeefc19c64d51a9d533dce0342b60 2024-11-21T01:44:47,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/cd6aeefc19c64d51a9d533dce0342b60, entries=200, sequenceid=426, filesize=39.0 K 2024-11-21T01:44:47,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/eb53cf430e1246f8858058fc4f0dd9e1 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/eb53cf430e1246f8858058fc4f0dd9e1 2024-11-21T01:44:47,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/eb53cf430e1246f8858058fc4f0dd9e1, entries=150, sequenceid=426, filesize=12.0 K 2024-11-21T01:44:47,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/00043a8198d84f7a93fcc0516d3c20fe as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/00043a8198d84f7a93fcc0516d3c20fe 2024-11-21T01:44:47,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/00043a8198d84f7a93fcc0516d3c20fe, entries=150, sequenceid=426, filesize=12.0 K 2024-11-21T01:44:47,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 5835087c277e0e7bdbee836efeecc2f0 in 959ms, sequenceid=426, compaction requested=true 2024-11-21T01:44:47,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:47,889 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:47,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5835087c277e0e7bdbee836efeecc2f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:44:47,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:47,890 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:47,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5835087c277e0e7bdbee836efeecc2f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:44:47,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:47,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5835087c277e0e7bdbee836efeecc2f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:44:47,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:47,890 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:47,890 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 5835087c277e0e7bdbee836efeecc2f0/A is initiating minor compaction (all files) 2024-11-21T01:44:47,890 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5835087c277e0e7bdbee836efeecc2f0/A in TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:47,891 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/438aa49f78054b138d5c0e67a11938c0, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/2a117c6f6ddc4fe4aa2b2db1b758eda8, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/cd6aeefc19c64d51a9d533dce0342b60] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp, totalSize=100.7 K 2024-11-21T01:44:47,891 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:47,891 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. files: [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/438aa49f78054b138d5c0e67a11938c0, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/2a117c6f6ddc4fe4aa2b2db1b758eda8, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/cd6aeefc19c64d51a9d533dce0342b60] 2024-11-21T01:44:47,891 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 438aa49f78054b138d5c0e67a11938c0, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732153482377 2024-11-21T01:44:47,891 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a117c6f6ddc4fe4aa2b2db1b758eda8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1732153483584 2024-11-21T01:44:47,892 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:47,892 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 5835087c277e0e7bdbee836efeecc2f0/B is initiating minor compaction (all files) 2024-11-21T01:44:47,892 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5835087c277e0e7bdbee836efeecc2f0/B in TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:47,892 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/d87d433f45124c2599db820c54e70073, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/6bf8f8ac0b5943b1ad6639101b7f490b, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/eb53cf430e1246f8858058fc4f0dd9e1] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp, totalSize=36.7 K 2024-11-21T01:44:47,892 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd6aeefc19c64d51a9d533dce0342b60, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1732153484747 2024-11-21T01:44:47,892 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting d87d433f45124c2599db820c54e70073, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732153482377 2024-11-21T01:44:47,893 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bf8f8ac0b5943b1ad6639101b7f490b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1732153483584 2024-11-21T01:44:47,893 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting eb53cf430e1246f8858058fc4f0dd9e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1732153484747 2024-11-21T01:44:47,900 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:47,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-21T01:44:47,904 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5835087c277e0e7bdbee836efeecc2f0#B#compaction#184 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:47,904 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/1b41d6b31de8453bade7f2bd2ae6c081 is 50, key is test_row_0/B:col10/1732153484747/Put/seqid=0 2024-11-21T01:44:47,914 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121eae956b95edd4f9cbd409319c29f9346_5835087c277e0e7bdbee836efeecc2f0 store=[table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:47,916 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121eae956b95edd4f9cbd409319c29f9346_5835087c277e0e7bdbee836efeecc2f0, store=[table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:47,916 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121eae956b95edd4f9cbd409319c29f9346_5835087c277e0e7bdbee836efeecc2f0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:47,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-21T01:44:47,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:47,925 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 5835087c277e0e7bdbee836efeecc2f0 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T01:44:47,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=A 2024-11-21T01:44:47,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:47,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=B 2024-11-21T01:44:47,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:47,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=C 2024-11-21T01:44:47,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:47,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742046_1222 (size=13085) 2024-11-21T01:44:47,995 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/1b41d6b31de8453bade7f2bd2ae6c081 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/1b41d6b31de8453bade7f2bd2ae6c081 2024-11-21T01:44:48,003 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5835087c277e0e7bdbee836efeecc2f0/B of 5835087c277e0e7bdbee836efeecc2f0 into 1b41d6b31de8453bade7f2bd2ae6c081(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:48,003 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:48,003 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., storeName=5835087c277e0e7bdbee836efeecc2f0/B, priority=13, startTime=1732153487889; duration=0sec 2024-11-21T01:44:48,003 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:48,003 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5835087c277e0e7bdbee836efeecc2f0:B 2024-11-21T01:44:48,003 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:48,006 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:48,006 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 5835087c277e0e7bdbee836efeecc2f0/C is initiating minor compaction (all files) 2024-11-21T01:44:48,006 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5835087c277e0e7bdbee836efeecc2f0/C in TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:48,006 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/d92c75c602b24413a1bcf556b8e92fb3, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/82eba9bc6e4f42ae9e69ca5e5dbce535, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/00043a8198d84f7a93fcc0516d3c20fe] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp, totalSize=36.7 K 2024-11-21T01:44:48,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742047_1223 (size=4469) 2024-11-21T01:44:48,007 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting d92c75c602b24413a1bcf556b8e92fb3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732153482377 2024-11-21T01:44:48,008 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 82eba9bc6e4f42ae9e69ca5e5dbce535, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1732153483584 2024-11-21T01:44:48,009 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 00043a8198d84f7a93fcc0516d3c20fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1732153484747 2024-11-21T01:44:48,009 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5835087c277e0e7bdbee836efeecc2f0#A#compaction#183 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:48,010 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/2a170c421f064fb4a52d62f65b912433 is 175, key is test_row_0/A:col10/1732153484747/Put/seqid=0 2024-11-21T01:44:48,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112154504fd6a83a4533bdf4faf9e27a5483_5835087c277e0e7bdbee836efeecc2f0 is 50, key is test_row_0/A:col10/1732153486973/Put/seqid=0 2024-11-21T01:44:48,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742048_1224 (size=32039) 2024-11-21T01:44:48,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742049_1225 (size=12454) 2024-11-21T01:44:48,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:48,056 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5835087c277e0e7bdbee836efeecc2f0#C#compaction#186 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:48,056 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/aeaffdec72e443f3853b7f7478133b6f is 50, key is test_row_0/C:col10/1732153484747/Put/seqid=0 2024-11-21T01:44:48,057 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/2a170c421f064fb4a52d62f65b912433 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/2a170c421f064fb4a52d62f65b912433 2024-11-21T01:44:48,063 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112154504fd6a83a4533bdf4faf9e27a5483_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112154504fd6a83a4533bdf4faf9e27a5483_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:48,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/78dcab3b01bb4842808d0adcb921319f, store: [table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:48,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/78dcab3b01bb4842808d0adcb921319f is 175, key is test_row_0/A:col10/1732153486973/Put/seqid=0 2024-11-21T01:44:48,084 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5835087c277e0e7bdbee836efeecc2f0/A of 5835087c277e0e7bdbee836efeecc2f0 into 2a170c421f064fb4a52d62f65b912433(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:48,084 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:48,084 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., storeName=5835087c277e0e7bdbee836efeecc2f0/A, priority=13, startTime=1732153487889; duration=0sec 2024-11-21T01:44:48,084 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:48,084 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5835087c277e0e7bdbee836efeecc2f0:A 2024-11-21T01:44:48,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742050_1226 (size=13085) 2024-11-21T01:44:48,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742051_1227 (size=31255) 2024-11-21T01:44:48,098 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/aeaffdec72e443f3853b7f7478133b6f as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/aeaffdec72e443f3853b7f7478133b6f 2024-11-21T01:44:48,099 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=446, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/78dcab3b01bb4842808d0adcb921319f 2024-11-21T01:44:48,112 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5835087c277e0e7bdbee836efeecc2f0/C of 5835087c277e0e7bdbee836efeecc2f0 into aeaffdec72e443f3853b7f7478133b6f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:48,112 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:48,112 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., storeName=5835087c277e0e7bdbee836efeecc2f0/C, priority=13, startTime=1732153487890; duration=0sec 2024-11-21T01:44:48,112 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:48,112 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5835087c277e0e7bdbee836efeecc2f0:C 2024-11-21T01:44:48,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/6b332516202a4eaeafdb797cfd5e24e0 is 50, key is test_row_0/B:col10/1732153486973/Put/seqid=0 2024-11-21T01:44:48,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:48,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. as already flushing 2024-11-21T01:44:48,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742052_1228 (size=12301) 2024-11-21T01:44:48,132 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/6b332516202a4eaeafdb797cfd5e24e0 2024-11-21T01:44:48,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:48,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42316 deadline: 1732153548138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:48,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/4348cedac5e04a49bf0e85082543c2b4 is 50, key is test_row_0/C:col10/1732153486973/Put/seqid=0 2024-11-21T01:44:48,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:48,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:48,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:48,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:48,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1732153548143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:48,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:48,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:48,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:48,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:48,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42302 deadline: 1732153548145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:48,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:48,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:48,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:48,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:48,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42352 deadline: 1732153548146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:48,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:48,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:48,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:44:48,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742053_1229 (size=12301) 2024-11-21T01:44:48,176 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/4348cedac5e04a49bf0e85082543c2b4 2024-11-21T01:44:48,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/78dcab3b01bb4842808d0adcb921319f as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/78dcab3b01bb4842808d0adcb921319f 2024-11-21T01:44:48,191 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/78dcab3b01bb4842808d0adcb921319f, entries=150, sequenceid=446, filesize=30.5 K 2024-11-21T01:44:48,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/6b332516202a4eaeafdb797cfd5e24e0 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/6b332516202a4eaeafdb797cfd5e24e0 2024-11-21T01:44:48,198 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/6b332516202a4eaeafdb797cfd5e24e0, entries=150, sequenceid=446, filesize=12.0 K 2024-11-21T01:44:48,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/4348cedac5e04a49bf0e85082543c2b4 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/4348cedac5e04a49bf0e85082543c2b4 2024-11-21T01:44:48,205 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/4348cedac5e04a49bf0e85082543c2b4, entries=150, sequenceid=446, filesize=12.0 K 2024-11-21T01:44:48,207 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 5835087c277e0e7bdbee836efeecc2f0 in 282ms, sequenceid=446, compaction requested=false 2024-11-21T01:44:48,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:48,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:48,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-21T01:44:48,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-21T01:44:48,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-21T01:44:48,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 616 msec 2024-11-21T01:44:48,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-21T01:44:48,215 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 629 msec 2024-11-21T01:44:48,215 INFO [Thread-663 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-663. 2024-11-21T01:44:48,216 DEBUG [Thread-663 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:44:48,216 DEBUG [Thread-663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:48,216 DEBUG [Thread-663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:48,216 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:44:48,217 INFO [Thread-657 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-657. 2024-11-21T01:44:48,217 DEBUG [Thread-657 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:44:48,217 DEBUG [Thread-657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:48,217 DEBUG [Thread-657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:48,218 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:44:48,220 INFO [Thread-661 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-661. 2024-11-21T01:44:48,220 DEBUG [Thread-661 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:44:48,220 DEBUG [Thread-661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:48,220 DEBUG [Thread-661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:48,220 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:44:48,220 INFO [Thread-659 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-659. 2024-11-21T01:44:48,220 DEBUG [Thread-659 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:44:48,220 DEBUG [Thread-659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:48,220 DEBUG [Thread-659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:48,220 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:44:48,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:48,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5835087c277e0e7bdbee836efeecc2f0 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T01:44:48,254 INFO [Thread-654 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-654. 2024-11-21T01:44:48,254 DEBUG [Thread-654 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:44:48,254 DEBUG [Thread-654 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:48,254 DEBUG [Thread-654 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:48,254 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:44:48,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=A 2024-11-21T01:44:48,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:48,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=B 2024-11-21T01:44:48,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:48,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=C 2024-11-21T01:44:48,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:48,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ffad8516b2df458fa8a92afca6b1b6ef_5835087c277e0e7bdbee836efeecc2f0 is 50, key is test_row_0/A:col10/1732153488253/Put/seqid=0 2024-11-21T01:44:48,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742054_1230 (size=12454) 2024-11-21T01:44:48,265 INFO [Thread-652 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-652. 2024-11-21T01:44:48,265 INFO [Thread-646 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-646. 2024-11-21T01:44:48,265 DEBUG [Thread-652 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:44:48,265 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:48,265 DEBUG [Thread-646 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:44:48,265 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:48,265 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:48,265 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:44:48,265 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:48,265 INFO [Thread-648 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-648. 2024-11-21T01:44:48,265 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:44:48,265 DEBUG [Thread-648 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:44:48,265 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:48,265 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:48,265 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:48,265 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:44:48,269 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ffad8516b2df458fa8a92afca6b1b6ef_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ffad8516b2df458fa8a92afca6b1b6ef_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:48,270 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/57584e6f45c347b7921b056cdb897438, store: [table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:48,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/57584e6f45c347b7921b056cdb897438 is 175, key is test_row_0/A:col10/1732153488253/Put/seqid=0 2024-11-21T01:44:48,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742055_1231 (size=31255) 2024-11-21T01:44:48,674 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=463, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/57584e6f45c347b7921b056cdb897438 2024-11-21T01:44:48,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/509db5ff76b44706b451b229903a44c1 is 50, key is test_row_0/B:col10/1732153488253/Put/seqid=0 2024-11-21T01:44:48,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742056_1232 (size=12301) 2024-11-21T01:44:48,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-21T01:44:48,723 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:44:49,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/509db5ff76b44706b451b229903a44c1 2024-11-21T01:44:49,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/9ff3c4e91a9c429c8c17d3770c5dd0c2 is 50, key is test_row_0/C:col10/1732153488253/Put/seqid=0 2024-11-21T01:44:49,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742057_1233 (size=12301) 2024-11-21T01:44:49,204 INFO [Thread-650 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-650. 2024-11-21T01:44:49,204 DEBUG [Thread-650 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:44:49,204 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:49,204 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:49,204 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:44:49,205 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T01:44:49,205 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-11-21T01:44:49,205 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-11-21T01:44:49,205 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-21T01:44:49,205 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-21T01:44:49,205 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-11-21T01:44:49,205 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T01:44:49,205 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2125 2024-11-21T01:44:49,205 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1959 2024-11-21T01:44:49,205 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T01:44:49,205 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 922 2024-11-21T01:44:49,205 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2766 rows 2024-11-21T01:44:49,205 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 928 2024-11-21T01:44:49,205 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2782 rows 2024-11-21T01:44:49,205 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T01:44:49,205 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T01:44:49,205 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:49,205 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:49,205 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:44:49,206 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-21T01:44:49,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T01:44:49,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-21T01:44:49,209 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732153489209"}]},"ts":"1732153489209"} 2024-11-21T01:44:49,210 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T01:44:49,210 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T01:44:49,211 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T01:44:49,212 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5835087c277e0e7bdbee836efeecc2f0, UNASSIGN}] 2024-11-21T01:44:49,213 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5835087c277e0e7bdbee836efeecc2f0, UNASSIGN 2024-11-21T01:44:49,214 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=5835087c277e0e7bdbee836efeecc2f0, regionState=CLOSING, regionLocation=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:49,215 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5835087c277e0e7bdbee836efeecc2f0, UNASSIGN because future has completed 2024-11-21T01:44:49,215 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T01:44:49,215 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359}] 2024-11-21T01:44:49,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-21T01:44:49,368 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(122): Close 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:49,368 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T01:44:49,368 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1722): Closing 5835087c277e0e7bdbee836efeecc2f0, disabling compactions & flushes 2024-11-21T01:44:49,368 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:49,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/9ff3c4e91a9c429c8c17d3770c5dd0c2 2024-11-21T01:44:49,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/57584e6f45c347b7921b056cdb897438 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/57584e6f45c347b7921b056cdb897438 2024-11-21T01:44:49,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/57584e6f45c347b7921b056cdb897438, entries=150, sequenceid=463, filesize=30.5 K 2024-11-21T01:44:49,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/509db5ff76b44706b451b229903a44c1 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/509db5ff76b44706b451b229903a44c1 2024-11-21T01:44:49,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/509db5ff76b44706b451b229903a44c1, entries=150, sequenceid=463, filesize=12.0 K 2024-11-21T01:44:49,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/9ff3c4e91a9c429c8c17d3770c5dd0c2 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/9ff3c4e91a9c429c8c17d3770c5dd0c2 2024-11-21T01:44:49,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/9ff3c4e91a9c429c8c17d3770c5dd0c2, entries=150, sequenceid=463, filesize=12.0 K 2024-11-21T01:44:49,521 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=26.84 KB/27480 for 5835087c277e0e7bdbee836efeecc2f0 in 1267ms, sequenceid=463, compaction requested=true 2024-11-21T01:44:49,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5835087c277e0e7bdbee836efeecc2f0: 2024-11-21T01:44:49,521 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:49,521 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:49,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5835087c277e0e7bdbee836efeecc2f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:44:49,521 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. after waiting 0 ms 2024-11-21T01:44:49,521 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:49,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:49,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5835087c277e0e7bdbee836efeecc2f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:44:49,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:49,521 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(2902): Flushing 5835087c277e0e7bdbee836efeecc2f0 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-21T01:44:49,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5835087c277e0e7bdbee836efeecc2f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:44:49,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:49,521 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. because compaction request was cancelled 2024-11-21T01:44:49,521 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. because compaction request was cancelled 2024-11-21T01:44:49,521 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5835087c277e0e7bdbee836efeecc2f0:A 2024-11-21T01:44:49,521 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5835087c277e0e7bdbee836efeecc2f0:B 2024-11-21T01:44:49,521 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. because compaction request was cancelled 2024-11-21T01:44:49,521 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5835087c277e0e7bdbee836efeecc2f0:C 2024-11-21T01:44:49,521 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=A 2024-11-21T01:44:49,521 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:49,521 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=B 2024-11-21T01:44:49,521 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:49,521 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5835087c277e0e7bdbee836efeecc2f0, store=C 2024-11-21T01:44:49,521 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:49,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-21T01:44:49,527 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121350dbc3279d9411c8ab9902fd26ea2d9_5835087c277e0e7bdbee836efeecc2f0 is 50, key is test_row_1/A:col10/1732153489203/Put/seqid=0 2024-11-21T01:44:49,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742058_1234 (size=9914) 2024-11-21T01:44:49,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-21T01:44:49,935 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:49,940 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121350dbc3279d9411c8ab9902fd26ea2d9_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121350dbc3279d9411c8ab9902fd26ea2d9_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:49,941 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/099f041ebf954e0dabd19420ca1c0dd8, store: [table=TestAcidGuarantees family=A region=5835087c277e0e7bdbee836efeecc2f0] 2024-11-21T01:44:49,941 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/099f041ebf954e0dabd19420ca1c0dd8 is 175, key is test_row_1/A:col10/1732153489203/Put/seqid=0 2024-11-21T01:44:49,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742059_1235 (size=22561) 2024-11-21T01:44:50,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-21T01:44:50,346 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=470, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/099f041ebf954e0dabd19420ca1c0dd8 2024-11-21T01:44:50,355 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/34996d51d4784dcdae324cd29df989c5 is 50, key is test_row_1/B:col10/1732153489203/Put/seqid=0 2024-11-21T01:44:50,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742060_1236 (size=9857) 2024-11-21T01:44:50,760 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/34996d51d4784dcdae324cd29df989c5 2024-11-21T01:44:50,769 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/9f37bc14c8284e3a845b86bfc4f3dc40 is 50, key is test_row_1/C:col10/1732153489203/Put/seqid=0 2024-11-21T01:44:50,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742061_1237 (size=9857) 2024-11-21T01:44:50,777 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/9f37bc14c8284e3a845b86bfc4f3dc40 2024-11-21T01:44:50,782 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/A/099f041ebf954e0dabd19420ca1c0dd8 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/099f041ebf954e0dabd19420ca1c0dd8 2024-11-21T01:44:50,786 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/099f041ebf954e0dabd19420ca1c0dd8, entries=100, sequenceid=470, filesize=22.0 K 2024-11-21T01:44:50,787 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/B/34996d51d4784dcdae324cd29df989c5 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/34996d51d4784dcdae324cd29df989c5 2024-11-21T01:44:50,791 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/34996d51d4784dcdae324cd29df989c5, entries=100, sequenceid=470, filesize=9.6 K 2024-11-21T01:44:50,798 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/.tmp/C/9f37bc14c8284e3a845b86bfc4f3dc40 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/9f37bc14c8284e3a845b86bfc4f3dc40 2024-11-21T01:44:50,802 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/9f37bc14c8284e3a845b86bfc4f3dc40, entries=100, sequenceid=470, filesize=9.6 K 2024-11-21T01:44:50,803 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 5835087c277e0e7bdbee836efeecc2f0 in 1282ms, sequenceid=470, compaction requested=true 2024-11-21T01:44:50,804 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/a8fdcb23f6fa4860b229720a998831d1, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/d4bac36c7bb7468f990eb3a9296a6b9c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/8fa01524a4554f9ea99178d70ad63255, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/1f9666f2e8234a938d7120b10677e400, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/4519ef66825c49c8ab6dee31a66bc435, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/4f4a27a074c94825bd8fd6dd6c4068dc, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/12ec4d42fd1845e5a16d7852fa976dde, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/abdef8ee57e0439a9fd03a53fe158c7d, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/acd27e1d2bc1479eb8b26424365dc0a2, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/bc6b7ba3300a488da3ba1b9ab5e8c88d, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/332657bf6e8d473dae18570fe91864c4, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/6a99845586d34d51b99a332aa8c78e10, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/bddb9bcae4124354966afc36bc8f29f1, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/89accf3e359d4f788f11bf2a357dd151, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/6387b83d41a24c01860ad5df5634260e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/9880acac652344c28ecd5044f9eb3a08, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/d291762108ab48a29abf8f0fbadf66f1, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/80f863c56cb34b3ebfee703776532e32, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/cad19ebcdfbe45a9a4aacbaeb4d15771, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/438aa49f78054b138d5c0e67a11938c0, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/440870c40e4e4c0285d42f913e567bc5, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/2a117c6f6ddc4fe4aa2b2db1b758eda8, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/cd6aeefc19c64d51a9d533dce0342b60] to archive 2024-11-21T01:44:50,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T01:44:50,807 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/a8fdcb23f6fa4860b229720a998831d1 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/a8fdcb23f6fa4860b229720a998831d1 2024-11-21T01:44:50,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/d4bac36c7bb7468f990eb3a9296a6b9c to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/d4bac36c7bb7468f990eb3a9296a6b9c 2024-11-21T01:44:50,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/8fa01524a4554f9ea99178d70ad63255 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/8fa01524a4554f9ea99178d70ad63255 2024-11-21T01:44:50,812 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/1f9666f2e8234a938d7120b10677e400 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/1f9666f2e8234a938d7120b10677e400 2024-11-21T01:44:50,813 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/4519ef66825c49c8ab6dee31a66bc435 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/4519ef66825c49c8ab6dee31a66bc435 2024-11-21T01:44:50,815 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/4f4a27a074c94825bd8fd6dd6c4068dc to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/4f4a27a074c94825bd8fd6dd6c4068dc 2024-11-21T01:44:50,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/12ec4d42fd1845e5a16d7852fa976dde to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/12ec4d42fd1845e5a16d7852fa976dde 2024-11-21T01:44:50,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/abdef8ee57e0439a9fd03a53fe158c7d to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/abdef8ee57e0439a9fd03a53fe158c7d 2024-11-21T01:44:50,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/acd27e1d2bc1479eb8b26424365dc0a2 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/acd27e1d2bc1479eb8b26424365dc0a2 2024-11-21T01:44:50,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/bc6b7ba3300a488da3ba1b9ab5e8c88d to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/bc6b7ba3300a488da3ba1b9ab5e8c88d 2024-11-21T01:44:50,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/332657bf6e8d473dae18570fe91864c4 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/332657bf6e8d473dae18570fe91864c4 2024-11-21T01:44:50,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/6a99845586d34d51b99a332aa8c78e10 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/6a99845586d34d51b99a332aa8c78e10 2024-11-21T01:44:50,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/bddb9bcae4124354966afc36bc8f29f1 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/bddb9bcae4124354966afc36bc8f29f1 2024-11-21T01:44:50,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/89accf3e359d4f788f11bf2a357dd151 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/89accf3e359d4f788f11bf2a357dd151 2024-11-21T01:44:50,827 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/6387b83d41a24c01860ad5df5634260e to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/6387b83d41a24c01860ad5df5634260e 2024-11-21T01:44:50,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/9880acac652344c28ecd5044f9eb3a08 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/9880acac652344c28ecd5044f9eb3a08 2024-11-21T01:44:50,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/d291762108ab48a29abf8f0fbadf66f1 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/d291762108ab48a29abf8f0fbadf66f1 2024-11-21T01:44:50,831 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/80f863c56cb34b3ebfee703776532e32 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/80f863c56cb34b3ebfee703776532e32 2024-11-21T01:44:50,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/cad19ebcdfbe45a9a4aacbaeb4d15771 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/cad19ebcdfbe45a9a4aacbaeb4d15771 2024-11-21T01:44:50,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/438aa49f78054b138d5c0e67a11938c0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/438aa49f78054b138d5c0e67a11938c0 2024-11-21T01:44:50,836 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/440870c40e4e4c0285d42f913e567bc5 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/440870c40e4e4c0285d42f913e567bc5 2024-11-21T01:44:50,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/2a117c6f6ddc4fe4aa2b2db1b758eda8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/2a117c6f6ddc4fe4aa2b2db1b758eda8 2024-11-21T01:44:50,839 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/cd6aeefc19c64d51a9d533dce0342b60 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/cd6aeefc19c64d51a9d533dce0342b60 2024-11-21T01:44:50,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/70e8711ecfaf4b7992f91cfce9e4743f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/3eae683dcae342568f5bee745339d7e1, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/6f3fe1feb9cf47edac999792b9b1cf5c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/4e4e0a20063e4c56bedea8e85e4aaf8e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/eca8558978b4455fb5aff4abfef0f8f1, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/2a68c74403ea4a5aa5d9233c15a16104, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/298dfd1916af4b248bcfbab29e595e03, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/746775a3eac546cda44f1e53195e7098, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/1e8a5d259c2b4050a5a3145b054fa602, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/5ac6db6aa8a24268aecd5c4a0f9888b3, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/a6fc4714f7ce474b8c82134eba1da198, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/5f07c54247bf4267b1068b0a865497f4, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/2e185f7e1dc944c6b3cc889f46d583b5, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/81315e934a6b42c1818d40bed066090a, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/402f0e3065964dabb46b5c99c53eab72, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/fe32b45b616d4ed39bd13e95d8a791ea, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/b9321bbf2a724fdd9b98d13adc0b32bd, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/d7acf32b9c3745aa80c9895e1cff4ae8, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/80c2bd3bf85d4d299d301a1d88e7ea91, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/d87d433f45124c2599db820c54e70073, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/07830ee979e1464384c5f150f4d6e848, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/6bf8f8ac0b5943b1ad6639101b7f490b, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/eb53cf430e1246f8858058fc4f0dd9e1] to archive 2024-11-21T01:44:50,842 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T01:44:50,844 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/70e8711ecfaf4b7992f91cfce9e4743f to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/70e8711ecfaf4b7992f91cfce9e4743f 2024-11-21T01:44:50,845 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/3eae683dcae342568f5bee745339d7e1 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/3eae683dcae342568f5bee745339d7e1 2024-11-21T01:44:50,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/6f3fe1feb9cf47edac999792b9b1cf5c to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/6f3fe1feb9cf47edac999792b9b1cf5c 2024-11-21T01:44:50,848 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/4e4e0a20063e4c56bedea8e85e4aaf8e to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/4e4e0a20063e4c56bedea8e85e4aaf8e 2024-11-21T01:44:50,849 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/eca8558978b4455fb5aff4abfef0f8f1 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/eca8558978b4455fb5aff4abfef0f8f1 2024-11-21T01:44:50,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/2a68c74403ea4a5aa5d9233c15a16104 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/2a68c74403ea4a5aa5d9233c15a16104 2024-11-21T01:44:50,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/298dfd1916af4b248bcfbab29e595e03 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/298dfd1916af4b248bcfbab29e595e03 2024-11-21T01:44:50,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/746775a3eac546cda44f1e53195e7098 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/746775a3eac546cda44f1e53195e7098 2024-11-21T01:44:50,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/1e8a5d259c2b4050a5a3145b054fa602 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/1e8a5d259c2b4050a5a3145b054fa602 2024-11-21T01:44:50,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/5ac6db6aa8a24268aecd5c4a0f9888b3 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/5ac6db6aa8a24268aecd5c4a0f9888b3 2024-11-21T01:44:50,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/a6fc4714f7ce474b8c82134eba1da198 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/a6fc4714f7ce474b8c82134eba1da198 2024-11-21T01:44:50,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/5f07c54247bf4267b1068b0a865497f4 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/5f07c54247bf4267b1068b0a865497f4 2024-11-21T01:44:50,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/2e185f7e1dc944c6b3cc889f46d583b5 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/2e185f7e1dc944c6b3cc889f46d583b5 2024-11-21T01:44:50,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/81315e934a6b42c1818d40bed066090a to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/81315e934a6b42c1818d40bed066090a 2024-11-21T01:44:50,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/402f0e3065964dabb46b5c99c53eab72 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/402f0e3065964dabb46b5c99c53eab72 2024-11-21T01:44:50,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/fe32b45b616d4ed39bd13e95d8a791ea to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/fe32b45b616d4ed39bd13e95d8a791ea 2024-11-21T01:44:50,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/b9321bbf2a724fdd9b98d13adc0b32bd to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/b9321bbf2a724fdd9b98d13adc0b32bd 2024-11-21T01:44:50,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/d7acf32b9c3745aa80c9895e1cff4ae8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/d7acf32b9c3745aa80c9895e1cff4ae8 2024-11-21T01:44:50,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/80c2bd3bf85d4d299d301a1d88e7ea91 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/80c2bd3bf85d4d299d301a1d88e7ea91 2024-11-21T01:44:50,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/d87d433f45124c2599db820c54e70073 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/d87d433f45124c2599db820c54e70073 2024-11-21T01:44:50,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/07830ee979e1464384c5f150f4d6e848 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/07830ee979e1464384c5f150f4d6e848 2024-11-21T01:44:50,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/6bf8f8ac0b5943b1ad6639101b7f490b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/6bf8f8ac0b5943b1ad6639101b7f490b 2024-11-21T01:44:50,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/eb53cf430e1246f8858058fc4f0dd9e1 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/eb53cf430e1246f8858058fc4f0dd9e1 2024-11-21T01:44:50,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/4f3f4859f83142af94aab4c72c128fd0, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/b53592f8387e46bb86b238df551b268b, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/202db00ad7f943c2a76b0d517125f96e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/a1a4f8126c3f42cd9a788c091561328c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/1374b3fca7c14cddb446e9ede797cb38, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/4ce634183b064f2f8946c2ad90cbce7e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/dc9502f1084d423195875dfbfe704095, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/d7b522a8644b401aac51ba34e69938b4, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/38869b7de0df4f66ac09aa35dd3dd0dc, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/201f9b8bd81b41fbbdff14fbd90de59f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/6405a90cc0cb417280b11a4a3ff2236f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/40650b2bbd2141c9a84c9398d151c210, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/49dedf7a6a134c7eae2550584cddaa31, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/cbc60e7b87494fc0bc4f38595c5562d9, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/25e885f22aae442fbefd41e5c0f82255, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/4b7129d62ee043959ecfb058ef2626eb, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/832655905ec443458dafddfcb3886734, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/5b661603cb0b473d98d599317856b027, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/04464739c90348e7bdd5a62bf9c94e0b, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/d92c75c602b24413a1bcf556b8e92fb3, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/33d5eac8aabb4e6c824789534e830f74, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/82eba9bc6e4f42ae9e69ca5e5dbce535, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/00043a8198d84f7a93fcc0516d3c20fe] to archive 2024-11-21T01:44:50,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T01:44:50,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/4f3f4859f83142af94aab4c72c128fd0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/4f3f4859f83142af94aab4c72c128fd0 2024-11-21T01:44:50,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/b53592f8387e46bb86b238df551b268b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/b53592f8387e46bb86b238df551b268b 2024-11-21T01:44:50,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/202db00ad7f943c2a76b0d517125f96e to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/202db00ad7f943c2a76b0d517125f96e 2024-11-21T01:44:50,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/a1a4f8126c3f42cd9a788c091561328c to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/a1a4f8126c3f42cd9a788c091561328c 2024-11-21T01:44:50,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/1374b3fca7c14cddb446e9ede797cb38 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/1374b3fca7c14cddb446e9ede797cb38 2024-11-21T01:44:50,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/4ce634183b064f2f8946c2ad90cbce7e to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/4ce634183b064f2f8946c2ad90cbce7e 2024-11-21T01:44:50,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/dc9502f1084d423195875dfbfe704095 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/dc9502f1084d423195875dfbfe704095 2024-11-21T01:44:50,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/d7b522a8644b401aac51ba34e69938b4 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/d7b522a8644b401aac51ba34e69938b4 2024-11-21T01:44:50,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/38869b7de0df4f66ac09aa35dd3dd0dc to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/38869b7de0df4f66ac09aa35dd3dd0dc 2024-11-21T01:44:50,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/201f9b8bd81b41fbbdff14fbd90de59f to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/201f9b8bd81b41fbbdff14fbd90de59f 2024-11-21T01:44:50,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/6405a90cc0cb417280b11a4a3ff2236f to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/6405a90cc0cb417280b11a4a3ff2236f 2024-11-21T01:44:50,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/40650b2bbd2141c9a84c9398d151c210 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/40650b2bbd2141c9a84c9398d151c210 2024-11-21T01:44:50,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/49dedf7a6a134c7eae2550584cddaa31 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/49dedf7a6a134c7eae2550584cddaa31 2024-11-21T01:44:50,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/cbc60e7b87494fc0bc4f38595c5562d9 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/cbc60e7b87494fc0bc4f38595c5562d9 2024-11-21T01:44:50,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/25e885f22aae442fbefd41e5c0f82255 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/25e885f22aae442fbefd41e5c0f82255 2024-11-21T01:44:50,900 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/4b7129d62ee043959ecfb058ef2626eb to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/4b7129d62ee043959ecfb058ef2626eb 2024-11-21T01:44:50,902 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/832655905ec443458dafddfcb3886734 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/832655905ec443458dafddfcb3886734 2024-11-21T01:44:50,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/5b661603cb0b473d98d599317856b027 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/5b661603cb0b473d98d599317856b027 2024-11-21T01:44:50,904 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/04464739c90348e7bdd5a62bf9c94e0b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/04464739c90348e7bdd5a62bf9c94e0b 2024-11-21T01:44:50,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/d92c75c602b24413a1bcf556b8e92fb3 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/d92c75c602b24413a1bcf556b8e92fb3 2024-11-21T01:44:50,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/33d5eac8aabb4e6c824789534e830f74 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/33d5eac8aabb4e6c824789534e830f74 2024-11-21T01:44:50,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/82eba9bc6e4f42ae9e69ca5e5dbce535 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/82eba9bc6e4f42ae9e69ca5e5dbce535 2024-11-21T01:44:50,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/00043a8198d84f7a93fcc0516d3c20fe to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/00043a8198d84f7a93fcc0516d3c20fe 2024-11-21T01:44:50,914 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/recovered.edits/473.seqid, newMaxSeqId=473, maxSeqId=4 2024-11-21T01:44:50,915 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. 2024-11-21T01:44:50,915 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1676): Region close journal for 5835087c277e0e7bdbee836efeecc2f0: Waiting for close lock at 1732153489368Running coprocessor pre-close hooks at 1732153489368Disabling compacts and flushes for region at 1732153489368Disabling writes for close at 1732153489521 (+153 ms)Obtaining lock to block concurrent updates at 1732153489521Preparing flush snapshotting stores in 5835087c277e0e7bdbee836efeecc2f0 at 1732153489521Finished memstore snapshotting TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732153489521Flushing stores of TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0. at 1732153489522 (+1 ms)Flushing 5835087c277e0e7bdbee836efeecc2f0/A: creating writer at 1732153489522Flushing mob file 5835087c277e0e7bdbee836efeecc2f0/A: appending metadata at 1732153489527 (+5 ms)Flushing mob file 5835087c277e0e7bdbee836efeecc2f0/A: closing flushed file at 1732153489527Flushing 5835087c277e0e7bdbee836efeecc2f0/A: appending metadata at 1732153489941 (+414 ms)Flushing 5835087c277e0e7bdbee836efeecc2f0/A: closing flushed file at 1732153489941Flushing 5835087c277e0e7bdbee836efeecc2f0/B: creating writer at 1732153490351 (+410 ms)Flushing 5835087c277e0e7bdbee836efeecc2f0/B: appending metadata at 1732153490354 (+3 ms)Flushing 5835087c277e0e7bdbee836efeecc2f0/B: closing flushed file at 1732153490355 (+1 ms)Flushing 5835087c277e0e7bdbee836efeecc2f0/C: creating writer at 1732153490765 (+410 ms)Flushing 5835087c277e0e7bdbee836efeecc2f0/C: appending metadata at 1732153490768 (+3 ms)Flushing 5835087c277e0e7bdbee836efeecc2f0/C: closing flushed file at 1732153490768Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e853ba0: reopening flushed file at 1732153490781 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5bd09f46: reopening flushed file at 1732153490786 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50122582: reopening flushed file at 1732153490791 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 5835087c277e0e7bdbee836efeecc2f0 in 1282ms, sequenceid=470, compaction requested=true at 1732153490803 (+12 ms)Writing region close event to WAL at 1732153490910 (+107 ms)Running coprocessor post-close hooks at 1732153490915 (+5 ms)Closed at 1732153490915 2024-11-21T01:44:50,917 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(157): Closed 5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:50,918 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=5835087c277e0e7bdbee836efeecc2f0, regionState=CLOSED 2024-11-21T01:44:50,919 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 because future has completed 2024-11-21T01:44:50,921 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-21T01:44:50,921 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseRegionProcedure 5835087c277e0e7bdbee836efeecc2f0, server=b93ada835ef8,41129,1732153434359 in 1.7040 sec 2024-11-21T01:44:50,922 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-21T01:44:50,922 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5835087c277e0e7bdbee836efeecc2f0, UNASSIGN in 1.7090 sec 2024-11-21T01:44:50,925 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-21T01:44:50,925 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7120 sec 2024-11-21T01:44:50,926 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732153490926"}]},"ts":"1732153490926"} 2024-11-21T01:44:50,928 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T01:44:50,928 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T01:44:50,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7220 sec 2024-11-21T01:44:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-21T01:44:51,353 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:44:51,353 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-21T01:44:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:44:51,355 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:44:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T01:44:51,356 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=64, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:44:51,360 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:51,362 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A, FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B, FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C, FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/recovered.edits] 2024-11-21T01:44:51,365 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/099f041ebf954e0dabd19420ca1c0dd8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/099f041ebf954e0dabd19420ca1c0dd8 2024-11-21T01:44:51,367 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/2a170c421f064fb4a52d62f65b912433 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/2a170c421f064fb4a52d62f65b912433 2024-11-21T01:44:51,368 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/57584e6f45c347b7921b056cdb897438 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/57584e6f45c347b7921b056cdb897438 2024-11-21T01:44:51,369 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/78dcab3b01bb4842808d0adcb921319f to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/A/78dcab3b01bb4842808d0adcb921319f 2024-11-21T01:44:51,371 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/1b41d6b31de8453bade7f2bd2ae6c081 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/1b41d6b31de8453bade7f2bd2ae6c081 2024-11-21T01:44:51,372 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/34996d51d4784dcdae324cd29df989c5 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/34996d51d4784dcdae324cd29df989c5 2024-11-21T01:44:51,373 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/509db5ff76b44706b451b229903a44c1 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/509db5ff76b44706b451b229903a44c1 2024-11-21T01:44:51,375 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/6b332516202a4eaeafdb797cfd5e24e0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/B/6b332516202a4eaeafdb797cfd5e24e0 2024-11-21T01:44:51,377 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/4348cedac5e04a49bf0e85082543c2b4 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/4348cedac5e04a49bf0e85082543c2b4 2024-11-21T01:44:51,381 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/9f37bc14c8284e3a845b86bfc4f3dc40 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/9f37bc14c8284e3a845b86bfc4f3dc40 2024-11-21T01:44:51,386 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/9ff3c4e91a9c429c8c17d3770c5dd0c2 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/9ff3c4e91a9c429c8c17d3770c5dd0c2 2024-11-21T01:44:51,387 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/aeaffdec72e443f3853b7f7478133b6f to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/C/aeaffdec72e443f3853b7f7478133b6f 2024-11-21T01:44:51,390 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/recovered.edits/473.seqid to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0/recovered.edits/473.seqid 2024-11-21T01:44:51,390 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:51,391 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T01:44:51,391 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T01:44:51,392 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-21T01:44:51,397 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121350dbc3279d9411c8ab9902fd26ea2d9_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121350dbc3279d9411c8ab9902fd26ea2d9_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:51,398 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112138e14a9042c944a4b50643181dec3c49_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112138e14a9042c944a4b50643181dec3c49_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:51,399 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121413cb3d681f84c92a976634f41e7eb70_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121413cb3d681f84c92a976634f41e7eb70_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:51,400 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214e8a28f769624704a5c5e58758ae4dc6_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214e8a28f769624704a5c5e58758ae4dc6_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:51,402 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112154504fd6a83a4533bdf4faf9e27a5483_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112154504fd6a83a4533bdf4faf9e27a5483_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:51,403 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121567b7da6ae934287baf53b924940e836_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121567b7da6ae934287baf53b924940e836_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:51,405 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215b1dbb0803604e1690e2dfe6b53e03e8_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215b1dbb0803604e1690e2dfe6b53e03e8_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:51,406 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112175727f2334234b64af1e7d4a5dc194fe_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112175727f2334234b64af1e7d4a5dc194fe_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:51,408 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217c945ae3a3e741448a87426967efc010_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217c945ae3a3e741448a87426967efc010_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:51,409 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112197a93d369cc346fd9434d7e5ffd88f21_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112197a93d369cc346fd9434d7e5ffd88f21_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:51,410 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219c7ac4050abd4a05917bed57409da7ce_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219c7ac4050abd4a05917bed57409da7ce_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:51,412 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ad5dff4c5d254113845009ba857f0dcc_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ad5dff4c5d254113845009ba857f0dcc_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:51,413 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d4d109ba343f4f55ae38f1fbe96e99a0_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d4d109ba343f4f55ae38f1fbe96e99a0_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:51,414 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d684a9ae513f4f85bc96256a900f807b_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d684a9ae513f4f85bc96256a900f807b_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:51,415 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e2107c0268fe40a7b446369c63ed8ce3_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e2107c0268fe40a7b446369c63ed8ce3_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:51,417 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e4c1ed92dddb40ea9d37e31b0ce57c42_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e4c1ed92dddb40ea9d37e31b0ce57c42_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:51,418 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ea483e4f66484ebaa88eb84563253095_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ea483e4f66484ebaa88eb84563253095_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:51,419 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ed57002b59e14e31bfe58ce9a13cb93b_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ed57002b59e14e31bfe58ce9a13cb93b_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:51,420 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f2b95a139d614a71a42a06d50c6c8540_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f2b95a139d614a71a42a06d50c6c8540_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:51,421 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f3fdd3e444bc48cf9fdb3321dc561b09_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f3fdd3e444bc48cf9fdb3321dc561b09_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:51,422 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ffad8516b2df458fa8a92afca6b1b6ef_5835087c277e0e7bdbee836efeecc2f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ffad8516b2df458fa8a92afca6b1b6ef_5835087c277e0e7bdbee836efeecc2f0 2024-11-21T01:44:51,423 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T01:44:51,425 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=64, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:44:51,429 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T01:44:51,431 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T01:44:51,432 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=64, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:44:51,432 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T01:44:51,433 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732153491433"}]},"ts":"9223372036854775807"} 2024-11-21T01:44:51,435 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T01:44:51,435 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5835087c277e0e7bdbee836efeecc2f0, NAME => 'TestAcidGuarantees,,1732153465038.5835087c277e0e7bdbee836efeecc2f0.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T01:44:51,436 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T01:44:51,436 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732153491436"}]},"ts":"9223372036854775807"} 2024-11-21T01:44:51,438 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T01:44:51,438 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=64, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:44:51,440 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 85 msec 2024-11-21T01:44:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T01:44:51,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T01:44:51,463 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:44:51,473 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=257 (was 248) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/cluster_5ec1f663-147e-5567-4614-f2cd450fd939/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1139353644_22 at /127.0.0.1:55402 [Waiting for operation #556] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/cluster_5ec1f663-147e-5567-4614-f2cd450fd939/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1740360738_22 at /127.0.0.1:59750 [Waiting for operation #485] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1740360738_22 at /127.0.0.1:55350 [Waiting for operation #576] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1139353644_22 at /127.0.0.1:59740 [Waiting for operation #494] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=633 (was 608) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3174 (was 2900) - AvailableMemoryMB LEAK? - 2024-11-21T01:44:51,484 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=257, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=633, ProcessCount=11, AvailableMemoryMB=3173 2024-11-21T01:44:51,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T01:44:51,485 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T01:44:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T01:44:51,487 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T01:44:51,487 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:51,487 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 65 2024-11-21T01:44:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-21T01:44:51,488 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T01:44:51,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742062_1238 (size=1061) 2024-11-21T01:44:51,503 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d 2024-11-21T01:44:51,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742063_1239 (size=53) 2024-11-21T01:44:51,512 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:44:51,512 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 70b50c956032fda6af9bbc7628072175, disabling compactions & flushes 2024-11-21T01:44:51,512 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:51,512 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:51,512 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. after waiting 0 ms 2024-11-21T01:44:51,512 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:51,513 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:51,513 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 70b50c956032fda6af9bbc7628072175: Waiting for close lock at 1732153491512Disabling compacts and flushes for region at 1732153491512Disabling writes for close at 1732153491512Writing region close event to WAL at 1732153491512Closed at 1732153491512 2024-11-21T01:44:51,514 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T01:44:51,514 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732153491514"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732153491514"}]},"ts":"1732153491514"} 2024-11-21T01:44:51,515 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T01:44:51,516 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T01:44:51,516 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732153491516"}]},"ts":"1732153491516"} 2024-11-21T01:44:51,518 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T01:44:51,518 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=70b50c956032fda6af9bbc7628072175, ASSIGN}] 2024-11-21T01:44:51,519 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=70b50c956032fda6af9bbc7628072175, ASSIGN 2024-11-21T01:44:51,519 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=70b50c956032fda6af9bbc7628072175, ASSIGN; state=OFFLINE, location=b93ada835ef8,41129,1732153434359; forceNewPlan=false, retain=false 2024-11-21T01:44:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-21T01:44:51,670 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=70b50c956032fda6af9bbc7628072175, regionState=OPENING, regionLocation=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:51,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=70b50c956032fda6af9bbc7628072175, ASSIGN because future has completed 2024-11-21T01:44:51,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359}] 2024-11-21T01:44:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-21T01:44:51,827 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:51,827 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7752): Opening region: {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} 2024-11-21T01:44:51,828 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 70b50c956032fda6af9bbc7628072175 2024-11-21T01:44:51,828 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:44:51,828 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7794): checking encryption for 70b50c956032fda6af9bbc7628072175 2024-11-21T01:44:51,828 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7797): checking classloading for 70b50c956032fda6af9bbc7628072175 2024-11-21T01:44:51,829 INFO [StoreOpener-70b50c956032fda6af9bbc7628072175-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 70b50c956032fda6af9bbc7628072175 2024-11-21T01:44:51,831 INFO [StoreOpener-70b50c956032fda6af9bbc7628072175-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T01:44:51,831 INFO [StoreOpener-70b50c956032fda6af9bbc7628072175-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 70b50c956032fda6af9bbc7628072175 columnFamilyName A 2024-11-21T01:44:51,831 DEBUG [StoreOpener-70b50c956032fda6af9bbc7628072175-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:51,831 INFO [StoreOpener-70b50c956032fda6af9bbc7628072175-1 {}] regionserver.HStore(327): Store=70b50c956032fda6af9bbc7628072175/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:44:51,831 INFO [StoreOpener-70b50c956032fda6af9bbc7628072175-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 70b50c956032fda6af9bbc7628072175 2024-11-21T01:44:51,832 INFO [StoreOpener-70b50c956032fda6af9bbc7628072175-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T01:44:51,832 INFO [StoreOpener-70b50c956032fda6af9bbc7628072175-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 70b50c956032fda6af9bbc7628072175 columnFamilyName B 2024-11-21T01:44:51,833 DEBUG [StoreOpener-70b50c956032fda6af9bbc7628072175-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:51,833 INFO [StoreOpener-70b50c956032fda6af9bbc7628072175-1 {}] regionserver.HStore(327): Store=70b50c956032fda6af9bbc7628072175/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:44:51,833 INFO [StoreOpener-70b50c956032fda6af9bbc7628072175-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 70b50c956032fda6af9bbc7628072175 2024-11-21T01:44:51,834 INFO [StoreOpener-70b50c956032fda6af9bbc7628072175-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T01:44:51,834 INFO [StoreOpener-70b50c956032fda6af9bbc7628072175-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 70b50c956032fda6af9bbc7628072175 columnFamilyName C 2024-11-21T01:44:51,834 DEBUG [StoreOpener-70b50c956032fda6af9bbc7628072175-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:44:51,834 INFO [StoreOpener-70b50c956032fda6af9bbc7628072175-1 {}] regionserver.HStore(327): Store=70b50c956032fda6af9bbc7628072175/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:44:51,835 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:51,835 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1038): replaying wal for 70b50c956032fda6af9bbc7628072175 2024-11-21T01:44:51,836 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175 2024-11-21T01:44:51,836 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175 2024-11-21T01:44:51,837 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1048): stopping wal replay for 70b50c956032fda6af9bbc7628072175 2024-11-21T01:44:51,837 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1060): Cleaning up temporary data for 70b50c956032fda6af9bbc7628072175 2024-11-21T01:44:51,838 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T01:44:51,839 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1093): writing seq id for 70b50c956032fda6af9bbc7628072175 2024-11-21T01:44:51,841 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T01:44:51,842 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1114): Opened 70b50c956032fda6af9bbc7628072175; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69229018, jitterRate=0.031592756509780884}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T01:44:51,842 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 70b50c956032fda6af9bbc7628072175 2024-11-21T01:44:51,842 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1006): Region open journal for 70b50c956032fda6af9bbc7628072175: Running coprocessor pre-open hook at 1732153491828Writing region info on filesystem at 1732153491828Initializing all the Stores at 1732153491829 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732153491829Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732153491829Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732153491829Cleaning up temporary data from old regions at 1732153491837 (+8 ms)Running coprocessor post-open hooks at 1732153491842 (+5 ms)Region opened successfully at 1732153491842 2024-11-21T01:44:51,843 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., pid=67, masterSystemTime=1732153491824 2024-11-21T01:44:51,845 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:51,845 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:51,845 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=70b50c956032fda6af9bbc7628072175, regionState=OPEN, openSeqNum=2, regionLocation=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:51,846 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 because future has completed 2024-11-21T01:44:51,849 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-21T01:44:51,849 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure 70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 in 175 msec 2024-11-21T01:44:51,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-21T01:44:51,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=70b50c956032fda6af9bbc7628072175, ASSIGN in 331 msec 2024-11-21T01:44:51,852 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T01:44:51,852 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732153491852"}]},"ts":"1732153491852"} 2024-11-21T01:44:51,853 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T01:44:51,854 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T01:44:51,855 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 369 msec 2024-11-21T01:44:52,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-21T01:44:52,113 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:44:52,114 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c45ccf4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,114 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:44:52,114 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:44:52,115 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:44:52,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:44:52,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:44:52,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@318d70f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:44:52,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:44:52,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:52,117 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59804, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:44:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5826da06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,118 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:44:52,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:44:52,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:44:52,120 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54036, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:44:52,121 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T01:44:52,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is b93ada835ef8,36607,1732153433607 2024-11-21T01:44:52,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@62734d81 2024-11-21T01:44:52,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T01:44:52,123 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59812, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T01:44:52,124 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e140226, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,124 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:44:52,124 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:44:52,125 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:44:52,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:44:52,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:44:52,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f7c75e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:44:52,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:44:52,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:52,126 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59834, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:44:52,127 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@273e7d09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,129 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@279bf308, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,129 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:44:52,130 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:44:52,130 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:44:52,130 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:44:52,130 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:44:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@241dfc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:44:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:44:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:52,132 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59842, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:44:52,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff6dbdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,141 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ba0760, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,141 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:44:52,141 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:44:52,142 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:44:52,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:44:52,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:44:52,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a57b91f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:44:52,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:44:52,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:52,144 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:44:52,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d9ec43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,145 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75f8f105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,145 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:44:52,145 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:44:52,146 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:44:52,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:44:52,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:44:52,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ca626af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:44:52,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:44:52,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:52,147 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59866, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:44:52,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff7d0bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,153 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aaa706, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,153 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:44:52,153 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:44:52,154 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:44:52,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:44:52,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:44:52,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ad6b623, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:44:52,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:44:52,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:52,156 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:44:52,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e6ecc71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,157 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@417b74fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,157 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:44:52,157 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:44:52,157 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:44:52,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:44:52,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:44:52,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6385879c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:44:52,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:44:52,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:52,159 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59900, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:44:52,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1898e6ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e88d13e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,160 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:44:52,160 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:44:52,161 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:44:52,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:44:52,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:44:52,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@207ea0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:44:52,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:44:52,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:52,162 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59914, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:44:52,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@717bba86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,163 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62fcffc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,163 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:44:52,163 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:44:52,164 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:44:52,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:44:52,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:44:52,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5002b38b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:44:52,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:44:52,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:52,165 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:44:52,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11e59b8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,166 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d83ce24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,166 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:44:52,166 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:44:52,167 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:44:52,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:44:52,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:44:52,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@714ff7a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:44:52,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:44:52,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:52,168 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59964, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:44:52,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bd84749, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,169 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1591929c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,169 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:44:52,169 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:44:52,170 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:44:52,170 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:44:52,170 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:44:52,170 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11693811, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,170 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:44:52,171 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:44:52,171 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:44:52,171 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59974, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:44:52,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fe7c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:44:52,173 DEBUG [Thread-1115 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:44:52,173 DEBUG [Thread-1123 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:44:52,173 DEBUG [Thread-1117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:44:52,173 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:44:52,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:44:52,174 DEBUG [Thread-1132 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:44:52,174 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:44:52,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:44:52,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:44:52,174 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:44:52,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:44:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-21T01:44:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T01:44:52,175 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:44:52,175 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:44:52,176 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:44:52,176 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:44:52,176 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:44:52,176 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:44:52,177 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:44:52,177 DEBUG [Thread-1121 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:44:52,177 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:44:52,177 DEBUG [Thread-1134 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:44:52,177 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:44:52,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:44:52,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:44:52,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2] 2024-11-21T01:44:52,179 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:44:52,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2] 2024-11-21T01:44:52,179 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:44:52,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2] 2024-11-21T01:44:52,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2] 2024-11-21T01:44:52,181 DEBUG [Thread-1119 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:44:52,182 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54080, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:44:52,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:44:52,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:44:52,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2] 2024-11-21T01:44:52,183 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:44:52,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2] 2024-11-21T01:44:52,186 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54092, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:44:52,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 70b50c956032fda6af9bbc7628072175 2024-11-21T01:44:52,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70b50c956032fda6af9bbc7628072175 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T01:44:52,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=A 2024-11-21T01:44:52,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:52,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=B 2024-11-21T01:44:52,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:52,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=C 2024-11-21T01:44:52,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:52,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2] 2024-11-21T01:44:52,193 DEBUG [Thread-1130 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:44:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:44:52,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:44:52,197 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54096, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:44:52,198 DEBUG [Thread-1126 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:44:52,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2] 2024-11-21T01:44:52,201 DEBUG [Thread-1128 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:44:52,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:44:52,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:44:52,205 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54108, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:44:52,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:44:52,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:44:52,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2] 2024-11-21T01:44:52,208 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:44:52,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2] 2024-11-21T01:44:52,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/2057b9bedf78445f8c3f9832dd8b5d89 is 50, key is test_row_0/A:col10/1732153492186/Put/seqid=0 2024-11-21T01:44:52,232 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732153492210, max timestamp=1732153492228Num uniques 859; 2024-11-21T01:44:52,232 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T01:44:52,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732153492210, max timestamp=1732153492228Num uniques 859; 2024-11-21T01:44:52,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732153492210, max timestamp=1732153492228Num uniques 859; 2024-11-21T01:44:52,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T01:44:52,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T01:44:52,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742064_1240 (size=9657) 2024-11-21T01:44:52,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/2057b9bedf78445f8c3f9832dd8b5d89 2024-11-21T01:44:52,273 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732153492245, max timestamp=1732153492262Num uniques 1018; 2024-11-21T01:44:52,273 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-21T01:44:52,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732153492245, max timestamp=1732153492262Num uniques 1018; 2024-11-21T01:44:52,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-21T01:44:52,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732153492245, max timestamp=1732153492262Num uniques 1018; 2024-11-21T01:44:52,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-21T01:44:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T01:44:52,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/6386a7cdd69340b8ac333bb0575c9f5f is 50, key is test_row_0/B:col10/1732153492186/Put/seqid=0 2024-11-21T01:44:52,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153552300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:52,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:52,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:52,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153552306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:52,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153552308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:52,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:52,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153552310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:52,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:52,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:52,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742065_1241 (size=9657) 2024-11-21T01:44:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153552311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:52,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:52,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/6386a7cdd69340b8ac333bb0575c9f5f 2024-11-21T01:44:52,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T01:44:52,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:52,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:44:52,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:52,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:52,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:52,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/9217a03ebff34efc8eb0c1105649a884 is 50, key is test_row_0/C:col10/1732153492186/Put/seqid=0 2024-11-21T01:44:52,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742066_1242 (size=9657) 2024-11-21T01:44:52,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/9217a03ebff34efc8eb0c1105649a884 2024-11-21T01:44:52,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/2057b9bedf78445f8c3f9832dd8b5d89 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/2057b9bedf78445f8c3f9832dd8b5d89 2024-11-21T01:44:52,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/2057b9bedf78445f8c3f9832dd8b5d89, entries=100, sequenceid=11, filesize=9.4 K 2024-11-21T01:44:52,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/6386a7cdd69340b8ac333bb0575c9f5f as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/6386a7cdd69340b8ac333bb0575c9f5f 2024-11-21T01:44:52,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/6386a7cdd69340b8ac333bb0575c9f5f, entries=100, sequenceid=11, filesize=9.4 K 2024-11-21T01:44:52,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153552413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:52,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:52,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/9217a03ebff34efc8eb0c1105649a884 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/9217a03ebff34efc8eb0c1105649a884 2024-11-21T01:44:52,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/9217a03ebff34efc8eb0c1105649a884, entries=100, sequenceid=11, filesize=9.4 K 2024-11-21T01:44:52,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 70b50c956032fda6af9bbc7628072175 in 241ms, sequenceid=11, compaction requested=false 2024-11-21T01:44:52,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:52,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732153492296, max timestamp=1732153492433Num uniques 1077; 2024-11-21T01:44:52,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8093, totalHeapSize=18648, min timestamp=1732153492296, max timestamp=1732153492433Num uniques 1077; 2024-11-21T01:44:52,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-21T01:44:52,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49313, new segment data size=8093, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-21T01:44:52,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732153492296, max timestamp=1732153492433Num uniques 1077; 2024-11-21T01:44:52,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-21T01:44:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 70b50c956032fda6af9bbc7628072175 2024-11-21T01:44:52,446 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70b50c956032fda6af9bbc7628072175 3/3 column families, dataSize=53.67 KB heapSize=144.23 KB 2024-11-21T01:44:52,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=A 2024-11-21T01:44:52,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:52,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=B 2024-11-21T01:44:52,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:52,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=C 2024-11-21T01:44:52,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:52,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/0382280c8f654cf899c6fa3f9bc5a54e is 50, key is test_row_0/A:col10/1732153492446/Put/seqid=0 2024-11-21T01:44:52,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742067_1243 (size=12001) 2024-11-21T01:44:52,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/0382280c8f654cf899c6fa3f9bc5a54e 2024-11-21T01:44:52,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T01:44:52,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:52,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:44:52,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:52,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:52,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T01:44:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:52,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153552483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:52,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153552486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:52,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732153492469, max timestamp=1732153492491Num uniques 859; 2024-11-21T01:44:52,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T01:44:52,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732153492469, max timestamp=1732153492491Num uniques 859; 2024-11-21T01:44:52,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T01:44:52,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732153492469, max timestamp=1732153492491Num uniques 859; 2024-11-21T01:44:52,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T01:44:52,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:52,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:52,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/84c803a71ec7468f9ef44b65ae462a7d is 50, key is test_row_0/B:col10/1732153492446/Put/seqid=0 2024-11-21T01:44:52,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742068_1244 (size=12001) 2024-11-21T01:44:52,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/84c803a71ec7468f9ef44b65ae462a7d 2024-11-21T01:44:52,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/115201f373824d6abd3f91a320249a73 is 50, key is test_row_0/C:col10/1732153492446/Put/seqid=0 2024-11-21T01:44:52,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:52,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153552573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153552573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:52,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742069_1245 (size=12001) 2024-11-21T01:44:52,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/115201f373824d6abd3f91a320249a73 2024-11-21T01:44:52,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/0382280c8f654cf899c6fa3f9bc5a54e as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/0382280c8f654cf899c6fa3f9bc5a54e 2024-11-21T01:44:52,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/0382280c8f654cf899c6fa3f9bc5a54e, entries=150, sequenceid=71, filesize=11.7 K 2024-11-21T01:44:52,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/84c803a71ec7468f9ef44b65ae462a7d as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/84c803a71ec7468f9ef44b65ae462a7d 2024-11-21T01:44:52,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/84c803a71ec7468f9ef44b65ae462a7d, entries=150, sequenceid=71, filesize=11.7 K 2024-11-21T01:44:52,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/115201f373824d6abd3f91a320249a73 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/115201f373824d6abd3f91a320249a73 2024-11-21T01:44:52,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/115201f373824d6abd3f91a320249a73, entries=150, sequenceid=71, filesize=11.7 K 2024-11-21T01:44:52,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~164.11 KB/168048, currentSize=127.47 KB/130530 for 70b50c956032fda6af9bbc7628072175 in 171ms, sequenceid=71, compaction requested=false 2024-11-21T01:44:52,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 70b50c956032fda6af9bbc7628072175 2024-11-21T01:44:52,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70b50c956032fda6af9bbc7628072175 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-21T01:44:52,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=A 2024-11-21T01:44:52,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:52,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=B 2024-11-21T01:44:52,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:52,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=C 2024-11-21T01:44:52,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:52,629 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T01:44:52,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/bff4413918d043899da651dd861282b3 is 50, key is test_row_0/A:col10/1732153492618/Put/seqid=0 2024-11-21T01:44:52,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153552636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:52,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153552640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:52,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:52,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T01:44:52,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:52,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:44:52,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:52,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:52,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:52,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:52,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153552641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:52,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:52,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742070_1246 (size=16681) 2024-11-21T01:44:52,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/bff4413918d043899da651dd861282b3 2024-11-21T01:44:52,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:52,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153552693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:52,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:52,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153552693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:52,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:52,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:52,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/05af23124ea4486ab53ede1011059e65 is 50, key is test_row_0/B:col10/1732153492618/Put/seqid=0 2024-11-21T01:44:52,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742071_1247 (size=12001) 2024-11-21T01:44:52,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/05af23124ea4486ab53ede1011059e65 2024-11-21T01:44:52,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/cd8c40677dc44cb099f0b2a7ccf4326c is 50, key is test_row_0/C:col10/1732153492618/Put/seqid=0 2024-11-21T01:44:52,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:52,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153552752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:52,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:52,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:52,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153552753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:52,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:52,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153552766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:52,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:52,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742072_1248 (size=12001) 2024-11-21T01:44:52,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T01:44:52,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:52,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:44:52,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:52,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:52,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:52,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T01:44:52,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:52,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153552903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:52,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:52,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:52,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153552903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:52,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:52,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T01:44:52,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:52,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:44:52,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:52,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:52,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:52,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:52,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:52,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153552964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:52,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:52,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:52,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153552973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:52,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:52,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:52,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153552985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:52,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:52,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:53,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T01:44:53,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:53,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:44:53,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:53,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:53,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:53,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:53,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/cd8c40677dc44cb099f0b2a7ccf4326c 2024-11-21T01:44:53,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/bff4413918d043899da651dd861282b3 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/bff4413918d043899da651dd861282b3 2024-11-21T01:44:53,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/bff4413918d043899da651dd861282b3, entries=250, sequenceid=108, filesize=16.3 K 2024-11-21T01:44:53,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/05af23124ea4486ab53ede1011059e65 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/05af23124ea4486ab53ede1011059e65 2024-11-21T01:44:53,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/05af23124ea4486ab53ede1011059e65, entries=150, sequenceid=108, filesize=11.7 K 2024-11-21T01:44:53,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/cd8c40677dc44cb099f0b2a7ccf4326c as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/cd8c40677dc44cb099f0b2a7ccf4326c 2024-11-21T01:44:53,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:53,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153553213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:53,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:53,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:53,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:53,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:53,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153553213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:53,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/cd8c40677dc44cb099f0b2a7ccf4326c, entries=150, sequenceid=108, filesize=11.7 K 2024-11-21T01:44:53,222 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 70b50c956032fda6af9bbc7628072175 in 602ms, sequenceid=108, compaction requested=true 2024-11-21T01:44:53,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:53,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:44:53,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:53,223 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:53,223 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:53,225 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38339 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:53,225 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 70b50c956032fda6af9bbc7628072175/A is initiating minor compaction (all files) 2024-11-21T01:44:53,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:44:53,225 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70b50c956032fda6af9bbc7628072175/A in TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:53,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:53,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:44:53,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:53,225 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/2057b9bedf78445f8c3f9832dd8b5d89, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/0382280c8f654cf899c6fa3f9bc5a54e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/bff4413918d043899da651dd861282b3] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp, totalSize=37.4 K 2024-11-21T01:44:53,225 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:53,226 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 70b50c956032fda6af9bbc7628072175/B is initiating minor compaction (all files) 2024-11-21T01:44:53,226 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70b50c956032fda6af9bbc7628072175/B in TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:53,226 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/6386a7cdd69340b8ac333bb0575c9f5f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/84c803a71ec7468f9ef44b65ae462a7d, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/05af23124ea4486ab53ede1011059e65] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp, totalSize=32.9 K 2024-11-21T01:44:53,226 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2057b9bedf78445f8c3f9832dd8b5d89, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732153492180 2024-11-21T01:44:53,226 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 6386a7cdd69340b8ac333bb0575c9f5f, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732153492180 2024-11-21T01:44:53,227 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0382280c8f654cf899c6fa3f9bc5a54e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1732153492432 2024-11-21T01:44:53,227 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 84c803a71ec7468f9ef44b65ae462a7d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1732153492432 2024-11-21T01:44:53,227 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting bff4413918d043899da651dd861282b3, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732153492546 2024-11-21T01:44:53,227 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 05af23124ea4486ab53ede1011059e65, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732153492559 2024-11-21T01:44:53,238 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70b50c956032fda6af9bbc7628072175#B#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:53,239 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/19695532f5d14170901ab4aebeb265f4 is 50, key is test_row_0/B:col10/1732153492618/Put/seqid=0 2024-11-21T01:44:53,242 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70b50c956032fda6af9bbc7628072175#A#compaction#205 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:53,243 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/2a4d722b47b045e7b2b3eee325d9ab3f is 50, key is test_row_0/A:col10/1732153492618/Put/seqid=0 2024-11-21T01:44:53,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T01:44:53,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:53,267 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 70b50c956032fda6af9bbc7628072175 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-21T01:44:53,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=A 2024-11-21T01:44:53,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:53,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=B 2024-11-21T01:44:53,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:53,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=C 2024-11-21T01:44:53,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:53,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:44:53,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 70b50c956032fda6af9bbc7628072175 2024-11-21T01:44:53,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742073_1249 (size=12104) 2024-11-21T01:44:53,296 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/19695532f5d14170901ab4aebeb265f4 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/19695532f5d14170901ab4aebeb265f4 2024-11-21T01:44:53,304 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70b50c956032fda6af9bbc7628072175/B of 70b50c956032fda6af9bbc7628072175 into 19695532f5d14170901ab4aebeb265f4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:53,304 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:53,304 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., storeName=70b50c956032fda6af9bbc7628072175/B, priority=13, startTime=1732153493223; duration=0sec 2024-11-21T01:44:53,305 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:53,305 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:B 2024-11-21T01:44:53,305 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:53,306 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:53,306 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 70b50c956032fda6af9bbc7628072175/C is initiating minor compaction (all files) 2024-11-21T01:44:53,307 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70b50c956032fda6af9bbc7628072175/C in TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:53,307 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/9217a03ebff34efc8eb0c1105649a884, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/115201f373824d6abd3f91a320249a73, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/cd8c40677dc44cb099f0b2a7ccf4326c] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp, totalSize=32.9 K 2024-11-21T01:44:53,308 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 9217a03ebff34efc8eb0c1105649a884, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732153492180 2024-11-21T01:44:53,309 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 115201f373824d6abd3f91a320249a73, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1732153492432 2024-11-21T01:44:53,309 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting cd8c40677dc44cb099f0b2a7ccf4326c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732153492559 2024-11-21T01:44:53,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T01:44:53,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/dd5bd9b0d05e41c3be957c468cb82bc2 is 50, key is test_row_0/A:col10/1732153492639/Put/seqid=0 2024-11-21T01:44:53,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742074_1250 (size=12104) 2024-11-21T01:44:53,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732153493302, max timestamp=1732153493325Num uniques 859; 2024-11-21T01:44:53,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T01:44:53,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732153493302, max timestamp=1732153493325Num uniques 859; 2024-11-21T01:44:53,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T01:44:53,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732153493302, max timestamp=1732153493325Num uniques 859; 2024-11-21T01:44:53,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T01:44:53,335 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70b50c956032fda6af9bbc7628072175#C#compaction#207 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:53,336 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/c6d94948cbb148a398799781f3bae406 is 50, key is test_row_0/C:col10/1732153492618/Put/seqid=0 2024-11-21T01:44:53,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742075_1251 (size=9657) 2024-11-21T01:44:53,365 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/dd5bd9b0d05e41c3be957c468cb82bc2 2024-11-21T01:44:53,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742076_1252 (size=12104) 2024-11-21T01:44:53,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/7c133d3e231d4ff890ece2db513c17e9 is 50, key is test_row_0/B:col10/1732153492639/Put/seqid=0 2024-11-21T01:44:53,376 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/c6d94948cbb148a398799781f3bae406 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/c6d94948cbb148a398799781f3bae406 2024-11-21T01:44:53,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742077_1253 (size=9657) 2024-11-21T01:44:53,396 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/7c133d3e231d4ff890ece2db513c17e9 2024-11-21T01:44:53,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/150a426bb5fe4efebc2399a223021896 is 50, key is test_row_0/C:col10/1732153492639/Put/seqid=0 2024-11-21T01:44:53,407 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70b50c956032fda6af9bbc7628072175/C of 70b50c956032fda6af9bbc7628072175 into c6d94948cbb148a398799781f3bae406(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:53,407 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:53,407 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., storeName=70b50c956032fda6af9bbc7628072175/C, priority=13, startTime=1732153493225; duration=0sec 2024-11-21T01:44:53,407 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:53,407 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:C 2024-11-21T01:44:53,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742078_1254 (size=9657) 2024-11-21T01:44:53,409 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/150a426bb5fe4efebc2399a223021896 2024-11-21T01:44:53,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/dd5bd9b0d05e41c3be957c468cb82bc2 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/dd5bd9b0d05e41c3be957c468cb82bc2 2024-11-21T01:44:53,418 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/dd5bd9b0d05e41c3be957c468cb82bc2, entries=100, sequenceid=117, filesize=9.4 K 2024-11-21T01:44:53,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/7c133d3e231d4ff890ece2db513c17e9 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/7c133d3e231d4ff890ece2db513c17e9 2024-11-21T01:44:53,424 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/7c133d3e231d4ff890ece2db513c17e9, entries=100, sequenceid=117, filesize=9.4 K 2024-11-21T01:44:53,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/150a426bb5fe4efebc2399a223021896 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/150a426bb5fe4efebc2399a223021896 2024-11-21T01:44:53,429 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/150a426bb5fe4efebc2399a223021896, entries=100, sequenceid=117, filesize=9.4 K 2024-11-21T01:44:53,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:53,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153553421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:53,430 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 70b50c956032fda6af9bbc7628072175 in 163ms, sequenceid=117, compaction requested=false 2024-11-21T01:44:53,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:53,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:53,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-21T01:44:53,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:53,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-21T01:44:53,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:53,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:53,435 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-21T01:44:53,435 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2540 sec 2024-11-21T01:44:53,438 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.2640 sec 2024-11-21T01:44:53,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732153493393, max timestamp=1732153493431Num uniques 1068; 2024-11-21T01:44:53,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732153493393, max timestamp=1732153493431Num uniques 1068; 2024-11-21T01:44:53,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-21T01:44:53,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-21T01:44:53,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732153493393, max timestamp=1732153493431Num uniques 1068; 2024-11-21T01:44:53,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-21T01:44:53,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 70b50c956032fda6af9bbc7628072175 2024-11-21T01:44:53,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70b50c956032fda6af9bbc7628072175 3/3 column families, dataSize=53.67 KB heapSize=145.08 KB 2024-11-21T01:44:53,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=A 2024-11-21T01:44:53,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:53,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=B 2024-11-21T01:44:53,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:53,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=C 2024-11-21T01:44:53,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:53,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/29789c06a5854fc3984761431766df3f is 50, key is test_row_0/A:col10/1732153493433/Put/seqid=0 2024-11-21T01:44:53,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742079_1255 (size=16931) 2024-11-21T01:44:53,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/29789c06a5854fc3984761431766df3f 2024-11-21T01:44:53,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/35b62a1a6f774a2a9988dbe205bc46df is 50, key is test_row_0/B:col10/1732153493433/Put/seqid=0 2024-11-21T01:44:53,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742080_1256 (size=12151) 2024-11-21T01:44:53,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/35b62a1a6f774a2a9988dbe205bc46df 2024-11-21T01:44:53,520 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732153493499, max timestamp=1732153493517Num uniques 859; 2024-11-21T01:44:53,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T01:44:53,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732153493499, max timestamp=1732153493517Num uniques 859; 2024-11-21T01:44:53,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T01:44:53,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732153493499, max timestamp=1732153493517Num uniques 859; 2024-11-21T01:44:53,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T01:44:53,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/b778b520e41d48c6ba15268d16577a20 is 50, key is test_row_0/C:col10/1732153493433/Put/seqid=0 2024-11-21T01:44:53,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742081_1257 (size=12151) 2024-11-21T01:44:53,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/b778b520e41d48c6ba15268d16577a20 2024-11-21T01:44:53,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/29789c06a5854fc3984761431766df3f as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/29789c06a5854fc3984761431766df3f 2024-11-21T01:44:53,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/29789c06a5854fc3984761431766df3f, entries=250, sequenceid=162, filesize=16.5 K 2024-11-21T01:44:53,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/35b62a1a6f774a2a9988dbe205bc46df as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/35b62a1a6f774a2a9988dbe205bc46df 2024-11-21T01:44:53,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:53,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153553576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:53,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:53,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153553581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:53,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:53,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:53,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:53,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:53,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:53,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:53,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/35b62a1a6f774a2a9988dbe205bc46df, entries=150, sequenceid=162, filesize=11.9 K 2024-11-21T01:44:53,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:53,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153553602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:53,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:53,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:53,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:53,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/b778b520e41d48c6ba15268d16577a20 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/b778b520e41d48c6ba15268d16577a20 2024-11-21T01:44:53,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/b778b520e41d48c6ba15268d16577a20, entries=150, sequenceid=162, filesize=11.9 K 2024-11-21T01:44:53,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~164.95 KB/168912, currentSize=127.47 KB/130530 for 70b50c956032fda6af9bbc7628072175 in 183ms, sequenceid=162, compaction requested=true 2024-11-21T01:44:53,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:53,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:44:53,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:53,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:44:53,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:53,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:44:53,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T01:44:53,631 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-11-21T01:44:53,632 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-21T01:44:53,633 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-21T01:44:53,633 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. because compaction request was cancelled 2024-11-21T01:44:53,633 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:A 2024-11-21T01:44:53,633 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:53,643 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33912 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:53,643 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 70b50c956032fda6af9bbc7628072175/B is initiating minor compaction (all files) 2024-11-21T01:44:53,643 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70b50c956032fda6af9bbc7628072175/B in TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:53,643 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/19695532f5d14170901ab4aebeb265f4, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/7c133d3e231d4ff890ece2db513c17e9, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/35b62a1a6f774a2a9988dbe205bc46df] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp, totalSize=33.1 K 2024-11-21T01:44:53,645 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 19695532f5d14170901ab4aebeb265f4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732153492559 2024-11-21T01:44:53,647 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c133d3e231d4ff890ece2db513c17e9, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732153492638 2024-11-21T01:44:53,647 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 35b62a1a6f774a2a9988dbe205bc46df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732153493433 2024-11-21T01:44:53,671 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70b50c956032fda6af9bbc7628072175#B#compaction#213 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:53,671 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/9c4f04de156a43eaa506cfb5d49084be is 50, key is test_row_0/B:col10/1732153493433/Put/seqid=0 2024-11-21T01:44:53,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742082_1258 (size=12357) 2024-11-21T01:44:53,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 70b50c956032fda6af9bbc7628072175 2024-11-21T01:44:53,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70b50c956032fda6af9bbc7628072175 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-21T01:44:53,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=A 2024-11-21T01:44:53,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T01:44:53,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=B 2024-11-21T01:44:53,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T01:44:53,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=C 2024-11-21T01:44:53,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T01:44:53,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/16bf1b1610fb431e82ed24288be51fa3 is 50, key is test_row_0/A:col10/1732153493580/Put/seqid=0 2024-11-21T01:44:53,733 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/2a4d722b47b045e7b2b3eee325d9ab3f as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/2a4d722b47b045e7b2b3eee325d9ab3f 2024-11-21T01:44:53,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:53,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153553723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:53,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:53,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153553726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:53,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:53,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:53,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:53,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:53,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:53,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:53,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:53,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153553731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:53,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:53,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:53,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:53,745 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70b50c956032fda6af9bbc7628072175/A of 70b50c956032fda6af9bbc7628072175 into 2a4d722b47b045e7b2b3eee325d9ab3f(size=11.8 K), total size for store is 37.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:53,745 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:53,745 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., storeName=70b50c956032fda6af9bbc7628072175/A, priority=13, startTime=1732153493222; duration=0sec 2024-11-21T01:44:53,745 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:53,745 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:A 2024-11-21T01:44:53,745 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:53,746 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33912 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:53,746 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 70b50c956032fda6af9bbc7628072175/C is initiating minor compaction (all files) 2024-11-21T01:44:53,746 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70b50c956032fda6af9bbc7628072175/C in TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:53,746 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/c6d94948cbb148a398799781f3bae406, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/150a426bb5fe4efebc2399a223021896, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/b778b520e41d48c6ba15268d16577a20] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp, totalSize=33.1 K 2024-11-21T01:44:53,746 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6d94948cbb148a398799781f3bae406, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732153492559 2024-11-21T01:44:53,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:53,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153553734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:53,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:53,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153553739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:53,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:53,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:53,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:53,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:53,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:53,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:53,750 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 150a426bb5fe4efebc2399a223021896, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732153492638 2024-11-21T01:44:53,750 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting b778b520e41d48c6ba15268d16577a20, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732153493433 2024-11-21T01:44:53,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742083_1259 (size=14541) 2024-11-21T01:44:53,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/16bf1b1610fb431e82ed24288be51fa3 2024-11-21T01:44:53,780 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70b50c956032fda6af9bbc7628072175#C#compaction#215 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:53,781 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/2077a936b78d42638c94f15adfb22658 is 50, key is test_row_0/C:col10/1732153493433/Put/seqid=0 2024-11-21T01:44:53,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/2d988ab51a694698a71c88330ae7de7b is 50, key is test_row_0/B:col10/1732153493580/Put/seqid=0 2024-11-21T01:44:53,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742085_1261 (size=12151) 2024-11-21T01:44:53,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/2d988ab51a694698a71c88330ae7de7b 2024-11-21T01:44:53,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742084_1260 (size=12357) 2024-11-21T01:44:53,811 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/2077a936b78d42638c94f15adfb22658 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/2077a936b78d42638c94f15adfb22658 2024-11-21T01:44:53,817 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70b50c956032fda6af9bbc7628072175/C of 70b50c956032fda6af9bbc7628072175 into 2077a936b78d42638c94f15adfb22658(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:53,817 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:53,817 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., storeName=70b50c956032fda6af9bbc7628072175/C, priority=13, startTime=1732153493631; duration=0sec 2024-11-21T01:44:53,817 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:53,817 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:C 2024-11-21T01:44:53,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/f2d305ecdc424b8bb6f05cdc84bc5a7c is 50, key is test_row_0/C:col10/1732153493580/Put/seqid=0 2024-11-21T01:44:53,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:53,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153553845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:53,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:53,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:53,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:53,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:53,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153553853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:53,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:53,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:53,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:53,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742086_1262 (size=12151) 2024-11-21T01:44:53,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:53,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153553853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:53,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:53,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/f2d305ecdc424b8bb6f05cdc84bc5a7c 2024-11-21T01:44:53,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153553854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:53,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/16bf1b1610fb431e82ed24288be51fa3 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/16bf1b1610fb431e82ed24288be51fa3 2024-11-21T01:44:53,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/16bf1b1610fb431e82ed24288be51fa3, entries=200, sequenceid=200, filesize=14.2 K 2024-11-21T01:44:53,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/2d988ab51a694698a71c88330ae7de7b as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/2d988ab51a694698a71c88330ae7de7b 2024-11-21T01:44:53,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/2d988ab51a694698a71c88330ae7de7b, entries=150, sequenceid=200, filesize=11.9 K 2024-11-21T01:44:53,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/f2d305ecdc424b8bb6f05cdc84bc5a7c as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/f2d305ecdc424b8bb6f05cdc84bc5a7c 2024-11-21T01:44:53,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/f2d305ecdc424b8bb6f05cdc84bc5a7c, entries=150, sequenceid=200, filesize=11.9 K 2024-11-21T01:44:53,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 70b50c956032fda6af9bbc7628072175 in 175ms, sequenceid=200, compaction requested=true 2024-11-21T01:44:53,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:53,884 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:44:53,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:A, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:44:53,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:53,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:44:53,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:53,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:44:53,886 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53233 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:44:53,886 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 70b50c956032fda6af9bbc7628072175/A is initiating minor compaction (all files) 2024-11-21T01:44:53,886 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70b50c956032fda6af9bbc7628072175/A in TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:53,886 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/2a4d722b47b045e7b2b3eee325d9ab3f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/dd5bd9b0d05e41c3be957c468cb82bc2, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/29789c06a5854fc3984761431766df3f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/16bf1b1610fb431e82ed24288be51fa3] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp, totalSize=52.0 K 2024-11-21T01:44:53,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T01:44:53,887 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a4d722b47b045e7b2b3eee325d9ab3f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732153492559 2024-11-21T01:44:53,887 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd5bd9b0d05e41c3be957c468cb82bc2, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732153492638 2024-11-21T01:44:53,889 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29789c06a5854fc3984761431766df3f, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732153493433 2024-11-21T01:44:53,889 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16bf1b1610fb431e82ed24288be51fa3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732153493564 2024-11-21T01:44:53,924 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70b50c956032fda6af9bbc7628072175#A#compaction#218 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:53,925 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/b4f95049b08940ccab76bd0953671290 is 50, key is test_row_0/A:col10/1732153493580/Put/seqid=0 2024-11-21T01:44:53,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742087_1263 (size=12391) 2024-11-21T01:44:54,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 70b50c956032fda6af9bbc7628072175 2024-11-21T01:44:54,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70b50c956032fda6af9bbc7628072175 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T01:44:54,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=A 2024-11-21T01:44:54,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:54,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=B 2024-11-21T01:44:54,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:54,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=C 2024-11-21T01:44:54,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:54,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/138c318c37b24367bf98ef882ae3072a is 50, key is test_row_0/A:col10/1732153493719/Put/seqid=0 2024-11-21T01:44:54,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742088_1264 (size=14541) 2024-11-21T01:44:54,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/138c318c37b24367bf98ef882ae3072a 2024-11-21T01:44:54,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/bd59044c5fb248eab6aaaef4e76d6d35 is 50, key is test_row_0/B:col10/1732153493719/Put/seqid=0 2024-11-21T01:44:54,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732153494072, max timestamp=1732153494105Num uniques 859; 2024-11-21T01:44:54,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T01:44:54,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732153494072, max timestamp=1732153494105Num uniques 859; 2024-11-21T01:44:54,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732153494072, max timestamp=1732153494105Num uniques 859; 2024-11-21T01:44:54,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T01:44:54,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T01:44:54,113 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/9c4f04de156a43eaa506cfb5d49084be as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/9c4f04de156a43eaa506cfb5d49084be 2024-11-21T01:44:54,120 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70b50c956032fda6af9bbc7628072175/B of 70b50c956032fda6af9bbc7628072175 into 9c4f04de156a43eaa506cfb5d49084be(size=12.1 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:54,120 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:54,120 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., storeName=70b50c956032fda6af9bbc7628072175/B, priority=13, startTime=1732153493631; duration=0sec 2024-11-21T01:44:54,120 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T01:44:54,120 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:B 2024-11-21T01:44:54,120 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:B 2024-11-21T01:44:54,120 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-21T01:44:54,121 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-21T01:44:54,121 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-21T01:44:54,121 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. because compaction request was cancelled 2024-11-21T01:44:54,121 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:C 2024-11-21T01:44:54,121 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-21T01:44:54,121 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-21T01:44:54,122 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-21T01:44:54,122 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. because compaction request was cancelled 2024-11-21T01:44:54,122 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:B 2024-11-21T01:44:54,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742089_1265 (size=12151) 2024-11-21T01:44:54,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/bd59044c5fb248eab6aaaef4e76d6d35 2024-11-21T01:44:54,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/e6cfeca440e24507ac225f09036fa521 is 50, key is test_row_0/C:col10/1732153493719/Put/seqid=0 2024-11-21T01:44:54,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:54,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153554145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:54,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:54,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:54,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153554147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:54,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:54,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:54,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153554150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153554151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:54,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742090_1266 (size=12151) 2024-11-21T01:44:54,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/e6cfeca440e24507ac225f09036fa521 2024-11-21T01:44:54,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/138c318c37b24367bf98ef882ae3072a as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/138c318c37b24367bf98ef882ae3072a 2024-11-21T01:44:54,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/138c318c37b24367bf98ef882ae3072a, entries=200, sequenceid=212, filesize=14.2 K 2024-11-21T01:44:54,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/bd59044c5fb248eab6aaaef4e76d6d35 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/bd59044c5fb248eab6aaaef4e76d6d35 2024-11-21T01:44:54,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/bd59044c5fb248eab6aaaef4e76d6d35, entries=150, sequenceid=212, filesize=11.9 K 2024-11-21T01:44:54,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/e6cfeca440e24507ac225f09036fa521 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/e6cfeca440e24507ac225f09036fa521 2024-11-21T01:44:54,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/e6cfeca440e24507ac225f09036fa521, entries=150, sequenceid=212, filesize=11.9 K 2024-11-21T01:44:54,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=138.65 KB/141980 for 70b50c956032fda6af9bbc7628072175 in 128ms, sequenceid=212, compaction requested=true 2024-11-21T01:44:54,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:54,188 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 4 compacting, 1 eligible, 16 blocking 2024-11-21T01:44:54,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:44:54,188 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-21T01:44:54,188 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-21T01:44:54,188 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. because compaction request was cancelled 2024-11-21T01:44:54,188 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:A 2024-11-21T01:44:54,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:54,188 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:54,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:B, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:44:54,189 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:54,189 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 70b50c956032fda6af9bbc7628072175/B is initiating minor compaction (all files) 2024-11-21T01:44:54,189 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70b50c956032fda6af9bbc7628072175/B in TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:54,189 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/9c4f04de156a43eaa506cfb5d49084be, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/2d988ab51a694698a71c88330ae7de7b, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/bd59044c5fb248eab6aaaef4e76d6d35] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp, totalSize=35.8 K 2024-11-21T01:44:54,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:54,191 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c4f04de156a43eaa506cfb5d49084be, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732153493433 2024-11-21T01:44:54,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:C, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:44:54,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:54,191 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d988ab51a694698a71c88330ae7de7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732153493564 2024-11-21T01:44:54,192 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting bd59044c5fb248eab6aaaef4e76d6d35, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732153493719 2024-11-21T01:44:54,211 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70b50c956032fda6af9bbc7628072175#B#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:54,212 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/57bb121f562f40f9b90a7e8f9315792d is 50, key is test_row_0/B:col10/1732153493719/Put/seqid=0 2024-11-21T01:44:54,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742091_1267 (size=12459) 2024-11-21T01:44:54,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70b50c956032fda6af9bbc7628072175 3/3 column families, dataSize=145.36 KB heapSize=410.84 KB 2024-11-21T01:44:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 70b50c956032fda6af9bbc7628072175 2024-11-21T01:44:54,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732153494145, max timestamp=1732153494268Num uniques 1118; 2024-11-21T01:44:54,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-21T01:44:54,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732153494145, max timestamp=1732153494268Num uniques 1118; 2024-11-21T01:44:54,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-21T01:44:54,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=A 2024-11-21T01:44:54,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:54,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=B 2024-11-21T01:44:54,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:54,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=C 2024-11-21T01:44:54,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T01:44:54,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/b4741c5b1ec84516ab3091bb7e1c890c is 50, key is test_row_0/A:col10/1732153494268/Put/seqid=0 2024-11-21T01:44:54,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742092_1268 (size=14541) 2024-11-21T01:44:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T01:44:54,323 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:44:54,323 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:44:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-21T01:44:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T01:44:54,327 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:44:54,328 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:44:54,328 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:44:54,352 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/b4f95049b08940ccab76bd0953671290 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/b4f95049b08940ccab76bd0953671290 2024-11-21T01:44:54,361 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 70b50c956032fda6af9bbc7628072175/A of 70b50c956032fda6af9bbc7628072175 into b4f95049b08940ccab76bd0953671290(size=12.1 K), total size for store is 26.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:54,361 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:54,361 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., storeName=70b50c956032fda6af9bbc7628072175/A, priority=12, startTime=1732153493884; duration=0sec 2024-11-21T01:44:54,361 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:54,361 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:A 2024-11-21T01:44:54,361 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:54,363 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:54,363 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 70b50c956032fda6af9bbc7628072175/C is initiating minor compaction (all files) 2024-11-21T01:44:54,363 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70b50c956032fda6af9bbc7628072175/C in TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:54,363 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/2077a936b78d42638c94f15adfb22658, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/f2d305ecdc424b8bb6f05cdc84bc5a7c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/e6cfeca440e24507ac225f09036fa521] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp, totalSize=35.8 K 2024-11-21T01:44:54,363 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2077a936b78d42638c94f15adfb22658, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732153493433 2024-11-21T01:44:54,364 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2d305ecdc424b8bb6f05cdc84bc5a7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732153493564 2024-11-21T01:44:54,365 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6cfeca440e24507ac225f09036fa521, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732153493719 2024-11-21T01:44:54,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153554358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:54,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:54,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153554358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:54,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:54,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:54,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153554359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:54,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:54,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153554360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:54,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:54,375 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70b50c956032fda6af9bbc7628072175#C#compaction#224 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:54,375 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/ded6e9fdd7dc4b0e9237a09ea4fe31e9 is 50, key is test_row_0/C:col10/1732153493719/Put/seqid=0 2024-11-21T01:44:54,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742093_1269 (size=12459) 2024-11-21T01:44:54,399 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/ded6e9fdd7dc4b0e9237a09ea4fe31e9 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/ded6e9fdd7dc4b0e9237a09ea4fe31e9 2024-11-21T01:44:54,404 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70b50c956032fda6af9bbc7628072175/C of 70b50c956032fda6af9bbc7628072175 into ded6e9fdd7dc4b0e9237a09ea4fe31e9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:54,405 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:54,405 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., storeName=70b50c956032fda6af9bbc7628072175/C, priority=13, startTime=1732153494191; duration=0sec 2024-11-21T01:44:54,405 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:54,405 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:C 2024-11-21T01:44:54,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T01:44:54,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153554473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:54,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153554473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:54,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:54,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153554473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:54,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153554473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:54,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:54,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:54,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:54,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T01:44:54,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:54,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:44:54,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:54,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:54,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:54,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T01:44:54,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:54,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:44:54,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:54,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:54,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:54,639 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/57bb121f562f40f9b90a7e8f9315792d as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/57bb121f562f40f9b90a7e8f9315792d 2024-11-21T01:44:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T01:44:54,646 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70b50c956032fda6af9bbc7628072175/B of 70b50c956032fda6af9bbc7628072175 into 57bb121f562f40f9b90a7e8f9315792d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:54,646 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:54,646 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., storeName=70b50c956032fda6af9bbc7628072175/B, priority=13, startTime=1732153494188; duration=0sec 2024-11-21T01:44:54,646 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:54,646 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:B 2024-11-21T01:44:54,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153554687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:54,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:54,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153554687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:54,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:54,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:54,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153554687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:54,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:54,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:54,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/b4741c5b1ec84516ab3091bb7e1c890c 2024-11-21T01:44:54,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:54,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153554693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:54,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:54,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/f17cf062a61a4a7484ea4db6681ecb60 is 50, key is test_row_0/B:col10/1732153494268/Put/seqid=0 2024-11-21T01:44:54,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153554756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:54,759 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to b93ada835ef8,41129,1732153434359 for 'test_row_0' in 70b50c956032fda6af9bbc7628072175 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T01:44:54,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:54,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:54,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742094_1270 (size=12151) 2024-11-21T01:44:54,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T01:44:54,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:54,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:44:54,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:54,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:54,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:54,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T01:44:54,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:54,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:44:54,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:54,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:54,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T01:44:55,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153555003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:55,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:55,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153555004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:55,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:55,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:55,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:55,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153555004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:55,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:55,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:55,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153555004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:55,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:55,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:55,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:55,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:55,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:55,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:55,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:55,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:55,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T01:44:55,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:55,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:44:55,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:55,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:55,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:55,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/f17cf062a61a4a7484ea4db6681ecb60 2024-11-21T01:44:55,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/d5827c07617e44888ba305024b20cc38 is 50, key is test_row_0/C:col10/1732153494268/Put/seqid=0 2024-11-21T01:44:55,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742095_1271 (size=12151) 2024-11-21T01:44:55,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/d5827c07617e44888ba305024b20cc38 2024-11-21T01:44:55,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/b4741c5b1ec84516ab3091bb7e1c890c as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/b4741c5b1ec84516ab3091bb7e1c890c 2024-11-21T01:44:55,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/b4741c5b1ec84516ab3091bb7e1c890c, entries=200, sequenceid=252, filesize=14.2 K 2024-11-21T01:44:55,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/f17cf062a61a4a7484ea4db6681ecb60 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/f17cf062a61a4a7484ea4db6681ecb60 2024-11-21T01:44:55,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/f17cf062a61a4a7484ea4db6681ecb60, entries=150, sequenceid=252, filesize=11.9 K 2024-11-21T01:44:55,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/d5827c07617e44888ba305024b20cc38 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/d5827c07617e44888ba305024b20cc38 2024-11-21T01:44:55,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/d5827c07617e44888ba305024b20cc38, entries=150, sequenceid=252, filesize=11.9 K 2024-11-21T01:44:55,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T01:44:55,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:55,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:44:55,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:55,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:55,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:55,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:55,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~78.27 KB/80150, heapSize ~213.91 KB/219040, currentSize=107.34 KB/109920 for 70b50c956032fda6af9bbc7628072175 in 991ms, sequenceid=252, compaction requested=true 2024-11-21T01:44:55,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:55,262 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:55,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:44:55,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:55,262 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-21T01:44:55,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:44:55,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:55,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:44:55,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:55,264 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-21T01:44:55,265 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41473 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:55,265 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 70b50c956032fda6af9bbc7628072175/A is initiating minor compaction (all files) 2024-11-21T01:44:55,265 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70b50c956032fda6af9bbc7628072175/A in TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:55,265 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-21T01:44:55,265 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. because compaction request was cancelled 2024-11-21T01:44:55,265 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:B 2024-11-21T01:44:55,265 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-21T01:44:55,265 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/b4f95049b08940ccab76bd0953671290, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/138c318c37b24367bf98ef882ae3072a, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/b4741c5b1ec84516ab3091bb7e1c890c] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp, totalSize=40.5 K 2024-11-21T01:44:55,267 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4f95049b08940ccab76bd0953671290, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732153493564 2024-11-21T01:44:55,267 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-21T01:44:55,267 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-21T01:44:55,268 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. because compaction request was cancelled 2024-11-21T01:44:55,268 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:C 2024-11-21T01:44:55,268 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 138c318c37b24367bf98ef882ae3072a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732153493719 2024-11-21T01:44:55,269 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4741c5b1ec84516ab3091bb7e1c890c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732153494145 2024-11-21T01:44:55,280 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70b50c956032fda6af9bbc7628072175#A#compaction#227 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:55,281 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/0ba32bf22f494af8a278df47e993619a is 50, key is test_row_0/A:col10/1732153494268/Put/seqid=0 2024-11-21T01:44:55,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742096_1272 (size=12493) 2024-11-21T01:44:55,330 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/0ba32bf22f494af8a278df47e993619a as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/0ba32bf22f494af8a278df47e993619a 2024-11-21T01:44:55,336 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70b50c956032fda6af9bbc7628072175/A of 70b50c956032fda6af9bbc7628072175 into 0ba32bf22f494af8a278df47e993619a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:55,336 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:55,336 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., storeName=70b50c956032fda6af9bbc7628072175/A, priority=13, startTime=1732153495261; duration=0sec 2024-11-21T01:44:55,336 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:55,336 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:A 2024-11-21T01:44:55,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T01:44:55,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:55,413 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 70b50c956032fda6af9bbc7628072175 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T01:44:55,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=A 2024-11-21T01:44:55,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:55,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=B 2024-11-21T01:44:55,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:55,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=C 2024-11-21T01:44:55,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:55,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/ad16f5364b0c474bba146ceb06a5eba4 is 50, key is test_row_0/A:col10/1732153494356/Put/seqid=0 2024-11-21T01:44:55,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742097_1273 (size=12301) 2024-11-21T01:44:55,441 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/ad16f5364b0c474bba146ceb06a5eba4 2024-11-21T01:44:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T01:44:55,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/cf606f2b91de41648874c64b31c17eb6 is 50, key is test_row_0/B:col10/1732153494356/Put/seqid=0 2024-11-21T01:44:55,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742098_1274 (size=12301) 2024-11-21T01:44:55,492 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/cf606f2b91de41648874c64b31c17eb6 2024-11-21T01:44:55,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/b0c3ac6323b74fad9f0bf298f94d1cee is 50, key is test_row_0/C:col10/1732153494356/Put/seqid=0 2024-11-21T01:44:55,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:44:55,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 70b50c956032fda6af9bbc7628072175 2024-11-21T01:44:55,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153555530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:55,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153555530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:55,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:55,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153555534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:55,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153555536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:55,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742099_1275 (size=12301) 2024-11-21T01:44:55,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153555643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:55,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:55,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:55,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:55,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:55,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153555645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:55,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:55,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:55,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:55,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153555653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:55,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:55,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:55,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:55,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153555656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:55,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:55,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:55,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:55,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153555853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:55,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:55,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153555855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:55,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:55,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:55,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:55,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:55,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:55,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153555863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:55,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:55,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153555863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:55,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:55,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:55,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:55,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:55,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:55,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:55,955 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/b0c3ac6323b74fad9f0bf298f94d1cee 2024-11-21T01:44:55,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/ad16f5364b0c474bba146ceb06a5eba4 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/ad16f5364b0c474bba146ceb06a5eba4 2024-11-21T01:44:55,980 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/ad16f5364b0c474bba146ceb06a5eba4, entries=150, sequenceid=275, filesize=12.0 K 2024-11-21T01:44:55,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/cf606f2b91de41648874c64b31c17eb6 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/cf606f2b91de41648874c64b31c17eb6 2024-11-21T01:44:55,986 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/cf606f2b91de41648874c64b31c17eb6, entries=150, sequenceid=275, filesize=12.0 K 2024-11-21T01:44:55,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/b0c3ac6323b74fad9f0bf298f94d1cee as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/b0c3ac6323b74fad9f0bf298f94d1cee 2024-11-21T01:44:55,993 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/b0c3ac6323b74fad9f0bf298f94d1cee, entries=150, sequenceid=275, filesize=12.0 K 2024-11-21T01:44:55,994 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 70b50c956032fda6af9bbc7628072175 in 581ms, sequenceid=275, compaction requested=true 2024-11-21T01:44:55,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:55,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:55,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-21T01:44:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-21T01:44:55,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-21T01:44:55,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6680 sec 2024-11-21T01:44:56,010 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.6770 sec 2024-11-21T01:44:56,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 70b50c956032fda6af9bbc7628072175 2024-11-21T01:44:56,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70b50c956032fda6af9bbc7628072175 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T01:44:56,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=A 2024-11-21T01:44:56,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:56,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=B 2024-11-21T01:44:56,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:56,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=C 2024-11-21T01:44:56,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:56,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/3dac2a72b1d74c0aa24d1f881fde73de is 50, key is test_row_0/A:col10/1732153496168/Put/seqid=0 2024-11-21T01:44:56,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742100_1276 (size=12301) 2024-11-21T01:44:56,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/3dac2a72b1d74c0aa24d1f881fde73de 2024-11-21T01:44:56,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/a89663ac813c415bab1e88db50a4bcd9 is 50, key is test_row_0/B:col10/1732153496168/Put/seqid=0 2024-11-21T01:44:56,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:56,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153556200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:56,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:56,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:56,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:56,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153556207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:56,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:56,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:56,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:56,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153556209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:56,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:56,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:56,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:56,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:56,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153556210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:56,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:56,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:56,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:56,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742101_1277 (size=12301) 2024-11-21T01:44:56,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/a89663ac813c415bab1e88db50a4bcd9 2024-11-21T01:44:56,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/93bb8b71c2cc4c2c80612e5d0db7e636 is 50, key is test_row_0/C:col10/1732153496168/Put/seqid=0 2024-11-21T01:44:56,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742102_1278 (size=12301) 2024-11-21T01:44:56,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/93bb8b71c2cc4c2c80612e5d0db7e636 2024-11-21T01:44:56,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/3dac2a72b1d74c0aa24d1f881fde73de as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/3dac2a72b1d74c0aa24d1f881fde73de 2024-11-21T01:44:56,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/3dac2a72b1d74c0aa24d1f881fde73de, entries=150, sequenceid=290, filesize=12.0 K 2024-11-21T01:44:56,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/a89663ac813c415bab1e88db50a4bcd9 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/a89663ac813c415bab1e88db50a4bcd9 2024-11-21T01:44:56,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/a89663ac813c415bab1e88db50a4bcd9, entries=150, sequenceid=290, filesize=12.0 K 2024-11-21T01:44:56,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/93bb8b71c2cc4c2c80612e5d0db7e636 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/93bb8b71c2cc4c2c80612e5d0db7e636 2024-11-21T01:44:56,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/93bb8b71c2cc4c2c80612e5d0db7e636, entries=150, sequenceid=290, filesize=12.0 K 2024-11-21T01:44:56,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 70b50c956032fda6af9bbc7628072175 in 117ms, sequenceid=290, compaction requested=true 2024-11-21T01:44:56,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:56,285 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:56,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:44:56,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:56,286 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:44:56,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:44:56,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:56,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:44:56,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:56,287 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:56,287 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 70b50c956032fda6af9bbc7628072175/A is initiating minor compaction (all files) 2024-11-21T01:44:56,287 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70b50c956032fda6af9bbc7628072175/A in TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:56,287 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/0ba32bf22f494af8a278df47e993619a, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/ad16f5364b0c474bba146ceb06a5eba4, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/3dac2a72b1d74c0aa24d1f881fde73de] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp, totalSize=36.2 K 2024-11-21T01:44:56,287 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49212 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:44:56,287 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ba32bf22f494af8a278df47e993619a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732153494146 2024-11-21T01:44:56,287 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 70b50c956032fda6af9bbc7628072175/B is initiating minor compaction (all files) 2024-11-21T01:44:56,287 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70b50c956032fda6af9bbc7628072175/B in TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:56,287 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/57bb121f562f40f9b90a7e8f9315792d, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/f17cf062a61a4a7484ea4db6681ecb60, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/cf606f2b91de41648874c64b31c17eb6, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/a89663ac813c415bab1e88db50a4bcd9] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp, totalSize=48.1 K 2024-11-21T01:44:56,288 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 57bb121f562f40f9b90a7e8f9315792d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732153493719 2024-11-21T01:44:56,288 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad16f5364b0c474bba146ceb06a5eba4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732153494344 2024-11-21T01:44:56,288 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting f17cf062a61a4a7484ea4db6681ecb60, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732153494146 2024-11-21T01:44:56,289 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dac2a72b1d74c0aa24d1f881fde73de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732153495532 2024-11-21T01:44:56,289 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting cf606f2b91de41648874c64b31c17eb6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732153494344 2024-11-21T01:44:56,291 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting a89663ac813c415bab1e88db50a4bcd9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732153495532 2024-11-21T01:44:56,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 70b50c956032fda6af9bbc7628072175 2024-11-21T01:44:56,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70b50c956032fda6af9bbc7628072175 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T01:44:56,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732153496204, max timestamp=1732153496328Num uniques 859; 2024-11-21T01:44:56,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T01:44:56,332 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70b50c956032fda6af9bbc7628072175#A#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:56,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732153496204, max timestamp=1732153496329Num uniques 859; 2024-11-21T01:44:56,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=A 2024-11-21T01:44:56,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:56,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=B 2024-11-21T01:44:56,335 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70b50c956032fda6af9bbc7628072175#B#compaction#235 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:56,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732153496204, max timestamp=1732153496329Num uniques 859; 2024-11-21T01:44:56,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T01:44:56,336 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/027693d37f4f41f199cedf2b91239a3a is 50, key is test_row_0/A:col10/1732153496168/Put/seqid=0 2024-11-21T01:44:56,336 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T01:44:56,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:56,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=C 2024-11-21T01:44:56,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:56,337 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/007c916ae47f4694b2863afddc13f290 is 50, key is test_row_0/B:col10/1732153496168/Put/seqid=0 2024-11-21T01:44:56,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/37e87f4aa0b8459c87983f92a7935341 is 50, key is test_row_0/A:col10/1732153496329/Put/seqid=0 2024-11-21T01:44:56,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742103_1279 (size=12745) 2024-11-21T01:44:56,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742104_1280 (size=12745) 2024-11-21T01:44:56,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742105_1281 (size=12301) 2024-11-21T01:44:56,358 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/027693d37f4f41f199cedf2b91239a3a as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/027693d37f4f41f199cedf2b91239a3a 2024-11-21T01:44:56,371 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70b50c956032fda6af9bbc7628072175/A of 70b50c956032fda6af9bbc7628072175 into 027693d37f4f41f199cedf2b91239a3a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:56,371 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:56,371 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., storeName=70b50c956032fda6af9bbc7628072175/A, priority=13, startTime=1732153496285; duration=0sec 2024-11-21T01:44:56,371 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:56,371 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:A 2024-11-21T01:44:56,371 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:44:56,373 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49212 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:44:56,373 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 70b50c956032fda6af9bbc7628072175/C is initiating minor compaction (all files) 2024-11-21T01:44:56,373 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70b50c956032fda6af9bbc7628072175/C in TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:56,373 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/ded6e9fdd7dc4b0e9237a09ea4fe31e9, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/d5827c07617e44888ba305024b20cc38, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/b0c3ac6323b74fad9f0bf298f94d1cee, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/93bb8b71c2cc4c2c80612e5d0db7e636] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp, totalSize=48.1 K 2024-11-21T01:44:56,373 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting ded6e9fdd7dc4b0e9237a09ea4fe31e9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732153493719 2024-11-21T01:44:56,374 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5827c07617e44888ba305024b20cc38, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732153494146 2024-11-21T01:44:56,375 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0c3ac6323b74fad9f0bf298f94d1cee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732153494344 2024-11-21T01:44:56,375 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93bb8b71c2cc4c2c80612e5d0db7e636, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732153495532 2024-11-21T01:44:56,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732153496359, max timestamp=1732153496378Num uniques 859; 2024-11-21T01:44:56,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T01:44:56,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732153496359, max timestamp=1732153496378Num uniques 859; 2024-11-21T01:44:56,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T01:44:56,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732153496359, max timestamp=1732153496378Num uniques 859; 2024-11-21T01:44:56,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T01:44:56,405 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70b50c956032fda6af9bbc7628072175#C#compaction#237 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:44:56,408 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/1baf8419e9e64f0d928095d32b3d0acd is 50, key is test_row_0/C:col10/1732153496168/Put/seqid=0 2024-11-21T01:44:56,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742106_1282 (size=12745) 2024-11-21T01:44:56,418 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/1baf8419e9e64f0d928095d32b3d0acd as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/1baf8419e9e64f0d928095d32b3d0acd 2024-11-21T01:44:56,424 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 70b50c956032fda6af9bbc7628072175/C of 70b50c956032fda6af9bbc7628072175 into 1baf8419e9e64f0d928095d32b3d0acd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:56,424 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:56,424 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., storeName=70b50c956032fda6af9bbc7628072175/C, priority=12, startTime=1732153496286; duration=0sec 2024-11-21T01:44:56,424 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:56,424 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:C 2024-11-21T01:44:56,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732153496399, max timestamp=1732153496424Num uniques 1118; 2024-11-21T01:44:56,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-21T01:44:56,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732153496399, max timestamp=1732153496424Num uniques 1118; 2024-11-21T01:44:56,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-21T01:44:56,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732153496399, max timestamp=1732153496424Num uniques 1118; 2024-11-21T01:44:56,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-21T01:44:56,443 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T01:44:56,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T01:44:56,473 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:44:56,474 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:44:56,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-21T01:44:56,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T01:44:56,478 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:44:56,479 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:44:56,479 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:44:56,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153556494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:56,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:56,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:56,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:56,506 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732153496480, max timestamp=1732153496501Num uniques 1127; 2024-11-21T01:44:56,506 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8092, totalHeapSize=18648, min timestamp=1732153496480, max timestamp=1732153496501Num uniques 1127; 2024-11-21T01:44:56,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-21T01:44:56,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51602, new segment data size=8092, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-21T01:44:56,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732153496480, max timestamp=1732153496501Num uniques 1127; 2024-11-21T01:44:56,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=6870, suffix heap size=145176,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=150 2024-11-21T01:44:56,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732153496541, max timestamp=1732153496562Num uniques 1036; 2024-11-21T01:44:56,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-11-21T01:44:56,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732153496541, max timestamp=1732153496562Num uniques 1036; 2024-11-21T01:44:56,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-11-21T01:44:56,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732153496541, max timestamp=1732153496562Num uniques 1009; 2024-11-21T01:44:56,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=13096, suffix heap size=132904,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=286 2024-11-21T01:44:56,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T01:44:56,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:56,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153556611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:56,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:56,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153556613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:56,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:56,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:56,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:56,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732153496596, max timestamp=1732153496615Num uniques 1145; 2024-11-21T01:44:56,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=11220, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-11-21T01:44:56,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732153496596, max timestamp=1732153496615Num uniques 1145; 2024-11-21T01:44:56,621 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=11220, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-11-21T01:44:56,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:56,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:56,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:56,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732153496607, max timestamp=1732153496615Num uniques 1145; 2024-11-21T01:44:56,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=8930, suffix heap size=147048,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-11-21T01:44:56,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T01:44:56,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:56,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:44:56,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:56,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:56,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:56,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:56,683 DEBUG [regionserver/b93ada835ef8:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.07692307692307693, tune throughput to 53.85 MB/second 2024-11-21T01:44:56,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732153496679, max timestamp=1732153496695Num uniques 1104; 2024-11-21T01:44:56,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-21T01:44:56,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732153496679, max timestamp=1732153496695Num uniques 1054; 2024-11-21T01:44:56,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-21T01:44:56,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732153496679, max timestamp=1732153496695Num uniques 1104; 2024-11-21T01:44:56,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-21T01:44:56,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/37e87f4aa0b8459c87983f92a7935341 2024-11-21T01:44:56,765 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/007c916ae47f4694b2863afddc13f290 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/007c916ae47f4694b2863afddc13f290 2024-11-21T01:44:56,775 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 70b50c956032fda6af9bbc7628072175/B of 70b50c956032fda6af9bbc7628072175 into 007c916ae47f4694b2863afddc13f290(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:56,775 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:56,775 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., storeName=70b50c956032fda6af9bbc7628072175/B, priority=12, startTime=1732153496286; duration=0sec 2024-11-21T01:44:56,775 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:56,775 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:B 2024-11-21T01:44:56,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/850b1f47bc9c4ba0b2cbba78098f4807 is 50, key is test_row_0/B:col10/1732153496329/Put/seqid=0 2024-11-21T01:44:56,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T01:44:56,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:56,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:44:56,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:56,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:56,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:56,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:56,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T01:44:56,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732153496779, max timestamp=1732153496799Num uniques 1063; 2024-11-21T01:44:56,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-21T01:44:56,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732153496779, max timestamp=1732153496801Num uniques 1063; 2024-11-21T01:44:56,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-21T01:44:56,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732153496779, max timestamp=1732153496801Num uniques 1063; 2024-11-21T01:44:56,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-21T01:44:56,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742107_1283 (size=12301) 2024-11-21T01:44:56,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/850b1f47bc9c4ba0b2cbba78098f4807 2024-11-21T01:44:56,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/6ccde64db3764c9097d67359fdd3ba35 is 50, key is test_row_0/C:col10/1732153496329/Put/seqid=0 2024-11-21T01:44:56,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1732153496832, max timestamp=1732153496850Num uniques 1022; 2024-11-21T01:44:56,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-21T01:44:56,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1732153496832, max timestamp=1732153496850Num uniques 1022; 2024-11-21T01:44:56,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-21T01:44:56,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732153496832, max timestamp=1732153496850Num uniques 1022; 2024-11-21T01:44:56,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-21T01:44:56,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742108_1284 (size=12301) 2024-11-21T01:44:56,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/6ccde64db3764c9097d67359fdd3ba35 2024-11-21T01:44:56,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/37e87f4aa0b8459c87983f92a7935341 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/37e87f4aa0b8459c87983f92a7935341 2024-11-21T01:44:56,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:56,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153556894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:56,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1732153496887, max timestamp=1732153496901Num uniques 1081; 2024-11-21T01:44:56,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-21T01:44:56,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:56,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:56,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:56,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/37e87f4aa0b8459c87983f92a7935341, entries=150, sequenceid=312, filesize=12.0 K 2024-11-21T01:44:56,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/850b1f47bc9c4ba0b2cbba78098f4807 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/850b1f47bc9c4ba0b2cbba78098f4807 2024-11-21T01:44:56,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1732153496887, max timestamp=1732153496901Num uniques 1031; 2024-11-21T01:44:56,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-21T01:44:56,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1732153496887, max timestamp=1732153496901Num uniques 1081; 2024-11-21T01:44:56,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-21T01:44:56,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/850b1f47bc9c4ba0b2cbba78098f4807, entries=150, sequenceid=312, filesize=12.0 K 2024-11-21T01:44:56,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/6ccde64db3764c9097d67359fdd3ba35 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/6ccde64db3764c9097d67359fdd3ba35 2024-11-21T01:44:56,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/6ccde64db3764c9097d67359fdd3ba35, entries=150, sequenceid=312, filesize=12.0 K 2024-11-21T01:44:56,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T01:44:56,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:56,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:44:56,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:56,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:56,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:56,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:56,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~38.02 KB/38930, heapSize ~96.27 KB/98576, currentSize=46.96 KB/48090 for 70b50c956032fda6af9bbc7628072175 in 622ms, sequenceid=312, compaction requested=false 2024-11-21T01:44:56,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 70b50c956032fda6af9bbc7628072175 2024-11-21T01:44:56,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70b50c956032fda6af9bbc7628072175 3/3 column families, dataSize=53.67 KB heapSize=143.86 KB 2024-11-21T01:44:56,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=A 2024-11-21T01:44:56,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:56,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=B 2024-11-21T01:44:56,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:56,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=C 2024-11-21T01:44:56,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:44:56,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/e5665933e09a4d859da80c5343ca066c is 50, key is test_row_0/A:col10/1732153496907/Put/seqid=0 2024-11-21T01:44:56,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742109_1285 (size=12301) 2024-11-21T01:44:56,975 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/e5665933e09a4d859da80c5343ca066c 2024-11-21T01:44:56,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/0ca139f168504054836ad36110413a93 is 50, key is test_row_0/B:col10/1732153496907/Put/seqid=0 2024-11-21T01:44:56,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742110_1286 (size=12301) 2024-11-21T01:44:57,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153557031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:57,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:57,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153557075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:57,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:57,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:57,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153557075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:57,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153557075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:57,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:57,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:57,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153557082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:57,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:57,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T01:44:57,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:57,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:44:57,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:57,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:57,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T01:44:57,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:57,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153557192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153557193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:57,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:57,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:57,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153557193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:57,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153557193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:57,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153557193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:57,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T01:44:57,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:57,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:44:57,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:57,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:57,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:57,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/0ca139f168504054836ad36110413a93 2024-11-21T01:44:57,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/7b23ea0149d249069483e03639a973e3 is 50, key is test_row_0/C:col10/1732153496907/Put/seqid=0 2024-11-21T01:44:57,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T01:44:57,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:57,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:44:57,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:57,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:57,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:57,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153557403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:57,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153557406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:57,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:57,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153557406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:57,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:57,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153557406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:57,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153557408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:57,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:57,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:57,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:57,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742111_1287 (size=12301) 2024-11-21T01:44:57,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/7b23ea0149d249069483e03639a973e3 2024-11-21T01:44:57,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/e5665933e09a4d859da80c5343ca066c as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/e5665933e09a4d859da80c5343ca066c 2024-11-21T01:44:57,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/e5665933e09a4d859da80c5343ca066c, entries=150, sequenceid=479, filesize=12.0 K 2024-11-21T01:44:57,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/0ca139f168504054836ad36110413a93 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/0ca139f168504054836ad36110413a93 2024-11-21T01:44:57,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/0ca139f168504054836ad36110413a93, entries=150, sequenceid=479, filesize=12.0 K 2024-11-21T01:44:57,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/7b23ea0149d249069483e03639a973e3 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/7b23ea0149d249069483e03639a973e3 2024-11-21T01:44:57,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/7b23ea0149d249069483e03639a973e3, entries=150, sequenceid=479, filesize=12.0 K 2024-11-21T01:44:57,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~183.66 KB/188064, currentSize=114.05 KB/116790 for 70b50c956032fda6af9bbc7628072175 in 532ms, sequenceid=479, compaction requested=true 2024-11-21T01:44:57,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:57,488 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:57,489 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:57,489 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 70b50c956032fda6af9bbc7628072175/A is initiating minor compaction (all files) 2024-11-21T01:44:57,489 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70b50c956032fda6af9bbc7628072175/A in TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:57,489 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/027693d37f4f41f199cedf2b91239a3a, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/37e87f4aa0b8459c87983f92a7935341, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/e5665933e09a4d859da80c5343ca066c] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp, totalSize=36.5 K 2024-11-21T01:44:57,489 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 027693d37f4f41f199cedf2b91239a3a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732153495532 2024-11-21T01:44:57,490 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37e87f4aa0b8459c87983f92a7935341, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732153496204 2024-11-21T01:44:57,490 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5665933e09a4d859da80c5343ca066c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1732153496904 2024-11-21T01:44:57,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:44:57,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:57,496 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:57,497 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:57,498 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 70b50c956032fda6af9bbc7628072175/B is initiating minor compaction (all files) 2024-11-21T01:44:57,498 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70b50c956032fda6af9bbc7628072175/B in TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:57,498 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/007c916ae47f4694b2863afddc13f290, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/850b1f47bc9c4ba0b2cbba78098f4807, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/0ca139f168504054836ad36110413a93] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp, totalSize=36.5 K 2024-11-21T01:44:57,498 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 007c916ae47f4694b2863afddc13f290, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732153495532 2024-11-21T01:44:57,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:44:57,499 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 850b1f47bc9c4ba0b2cbba78098f4807, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732153496204 2024-11-21T01:44:57,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:57,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:44:57,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:57,499 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ca139f168504054836ad36110413a93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1732153496904 2024-11-21T01:44:57,501 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70b50c956032fda6af9bbc7628072175#A#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:44:57,501 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/e0dbfbe3180d484197bcfd7b8d19a05c is 50, key is test_row_0/A:col10/1732153496907/Put/seqid=0 2024-11-21T01:44:57,507 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70b50c956032fda6af9bbc7628072175#B#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:44:57,508 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/4295f090220140c29d189290084671ae is 50, key is test_row_0/B:col10/1732153496907/Put/seqid=0 2024-11-21T01:44:57,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742112_1288 (size=12847) 2024-11-21T01:44:57,532 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/e0dbfbe3180d484197bcfd7b8d19a05c as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/e0dbfbe3180d484197bcfd7b8d19a05c 2024-11-21T01:44:57,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742113_1289 (size=12847) 2024-11-21T01:44:57,552 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70b50c956032fda6af9bbc7628072175/A of 70b50c956032fda6af9bbc7628072175 into e0dbfbe3180d484197bcfd7b8d19a05c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:57,552 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:57,553 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., storeName=70b50c956032fda6af9bbc7628072175/A, priority=13, startTime=1732153497487; duration=0sec 2024-11-21T01:44:57,553 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:57,553 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:A 2024-11-21T01:44:57,553 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:57,554 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:57,554 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 70b50c956032fda6af9bbc7628072175/C is initiating minor compaction (all files) 2024-11-21T01:44:57,554 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70b50c956032fda6af9bbc7628072175/C in TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:57,554 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/1baf8419e9e64f0d928095d32b3d0acd, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/6ccde64db3764c9097d67359fdd3ba35, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/7b23ea0149d249069483e03639a973e3] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp, totalSize=36.5 K 2024-11-21T01:44:57,554 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1baf8419e9e64f0d928095d32b3d0acd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732153495532 2024-11-21T01:44:57,555 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ccde64db3764c9097d67359fdd3ba35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732153496204 2024-11-21T01:44:57,555 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b23ea0149d249069483e03639a973e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1732153496904 2024-11-21T01:44:57,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T01:44:57,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:57,570 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 70b50c956032fda6af9bbc7628072175 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T01:44:57,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=A 2024-11-21T01:44:57,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:57,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=B 2024-11-21T01:44:57,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:57,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=C 2024-11-21T01:44:57,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:57,579 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/4295f090220140c29d189290084671ae as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/4295f090220140c29d189290084671ae 2024-11-21T01:44:57,596 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70b50c956032fda6af9bbc7628072175/B of 70b50c956032fda6af9bbc7628072175 into 4295f090220140c29d189290084671ae(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:57,596 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:57,596 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., storeName=70b50c956032fda6af9bbc7628072175/B, priority=13, startTime=1732153497496; duration=0sec 2024-11-21T01:44:57,596 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:57,596 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:B 2024-11-21T01:44:57,596 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70b50c956032fda6af9bbc7628072175#C#compaction#245 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:44:57,597 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/2a6f9ae41de64a36bd2b63549a0ddd85 is 50, key is test_row_0/C:col10/1732153496907/Put/seqid=0 2024-11-21T01:44:57,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/aaa6db9d19874414a3dcd8001d264eee is 50, key is test_row_0/A:col10/1732153496968/Put/seqid=0 2024-11-21T01:44:57,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742114_1290 (size=12847) 2024-11-21T01:44:57,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T01:44:57,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742115_1291 (size=12301) 2024-11-21T01:44:57,615 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/aaa6db9d19874414a3dcd8001d264eee 2024-11-21T01:44:57,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/717b5232cc9b476b8710063459aca70e is 50, key is test_row_0/B:col10/1732153496968/Put/seqid=0 2024-11-21T01:44:57,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742116_1292 (size=12301) 2024-11-21T01:44:57,637 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/717b5232cc9b476b8710063459aca70e 2024-11-21T01:44:57,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/c4b010a4b9e643b1b0d7806538caa0f1 is 50, key is test_row_0/C:col10/1732153496968/Put/seqid=0 2024-11-21T01:44:57,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742117_1293 (size=12301) 2024-11-21T01:44:57,708 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/c4b010a4b9e643b1b0d7806538caa0f1 2024-11-21T01:44:57,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:44:57,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 70b50c956032fda6af9bbc7628072175 2024-11-21T01:44:57,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/aaa6db9d19874414a3dcd8001d264eee as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/aaa6db9d19874414a3dcd8001d264eee 2024-11-21T01:44:57,730 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/aaa6db9d19874414a3dcd8001d264eee, entries=150, sequenceid=500, filesize=12.0 K 2024-11-21T01:44:57,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/717b5232cc9b476b8710063459aca70e as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/717b5232cc9b476b8710063459aca70e 2024-11-21T01:44:57,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:57,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153557734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:57,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:57,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:57,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153557734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:57,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:57,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153557734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:57,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:57,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:57,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:57,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153557741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:57,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:57,745 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/717b5232cc9b476b8710063459aca70e, entries=150, sequenceid=500, filesize=12.0 K 2024-11-21T01:44:57,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/c4b010a4b9e643b1b0d7806538caa0f1 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/c4b010a4b9e643b1b0d7806538caa0f1 2024-11-21T01:44:57,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:57,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153557744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:57,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:57,751 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/c4b010a4b9e643b1b0d7806538caa0f1, entries=150, sequenceid=500, filesize=12.0 K 2024-11-21T01:44:57,753 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 70b50c956032fda6af9bbc7628072175 in 183ms, sequenceid=500, compaction requested=false 2024-11-21T01:44:57,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:57,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:57,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-21T01:44:57,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-21T01:44:57,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-21T01:44:57,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2750 sec 2024-11-21T01:44:57,770 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.2930 sec 2024-11-21T01:44:57,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 70b50c956032fda6af9bbc7628072175 2024-11-21T01:44:57,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70b50c956032fda6af9bbc7628072175 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T01:44:57,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=A 2024-11-21T01:44:57,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:57,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=B 2024-11-21T01:44:57,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:57,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=C 2024-11-21T01:44:57,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:57,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/46729d28357c42ba823d6d8356132dd5 is 50, key is test_row_0/A:col10/1732153497860/Put/seqid=0 2024-11-21T01:44:57,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:57,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153557890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:57,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:57,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153557891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:57,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:57,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153557893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:57,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:57,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:57,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153557893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:57,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:57,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:57,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:57,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:57,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153557894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:57,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:57,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742118_1294 (size=12301) 2024-11-21T01:44:57,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:58,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:58,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153558003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:58,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:58,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:58,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:58,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:58,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153558003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:58,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:58,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:58,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:58,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:58,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153558003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:58,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:58,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:58,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:58,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:58,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153558004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:58,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:58,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:58,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:58,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:58,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153558004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:58,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:58,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:58,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:58,018 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/2a6f9ae41de64a36bd2b63549a0ddd85 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/2a6f9ae41de64a36bd2b63549a0ddd85 2024-11-21T01:44:58,027 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70b50c956032fda6af9bbc7628072175/C of 70b50c956032fda6af9bbc7628072175 into 2a6f9ae41de64a36bd2b63549a0ddd85(size=12.5 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:58,027 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:58,027 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., storeName=70b50c956032fda6af9bbc7628072175/C, priority=13, startTime=1732153497499; duration=0sec 2024-11-21T01:44:58,027 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:58,027 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:C 2024-11-21T01:44:58,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:58,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153558213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:58,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:58,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:58,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:58,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:58,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153558213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:58,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:58,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:58,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:58,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:58,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153558213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:58,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:58,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:58,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:58,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153558214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:58,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:58,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:58,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153558214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:58,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:58,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:58,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:58,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:58,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:58,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:58,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/46729d28357c42ba823d6d8356132dd5 2024-11-21T01:44:58,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/4fb4f22e9f2644378c2e4d7d273e5577 is 50, key is test_row_0/B:col10/1732153497860/Put/seqid=0 2024-11-21T01:44:58,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742119_1295 (size=12301) 2024-11-21T01:44:58,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:58,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153558523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:58,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:58,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:58,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:58,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153558524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:58,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:58,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:58,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:58,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:58,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:58,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153558525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:58,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:58,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153558525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:58,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:58,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:58,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:58,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:58,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:58,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:58,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:58,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153558533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:58,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:58,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:58,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:58,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T01:44:58,623 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:44:58,623 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:44:58,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-21T01:44:58,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T01:44:58,626 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:44:58,627 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:44:58,627 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:44:58,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/4fb4f22e9f2644378c2e4d7d273e5577 2024-11-21T01:44:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T01:44:58,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/611c615ec5004ca48dc29bebf1dd428e is 50, key is test_row_0/C:col10/1732153497860/Put/seqid=0 2024-11-21T01:44:58,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742120_1296 (size=12301) 2024-11-21T01:44:58,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/611c615ec5004ca48dc29bebf1dd428e 2024-11-21T01:44:58,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/46729d28357c42ba823d6d8356132dd5 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/46729d28357c42ba823d6d8356132dd5 2024-11-21T01:44:58,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/46729d28357c42ba823d6d8356132dd5, entries=150, sequenceid=518, filesize=12.0 K 2024-11-21T01:44:58,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/4fb4f22e9f2644378c2e4d7d273e5577 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/4fb4f22e9f2644378c2e4d7d273e5577 2024-11-21T01:44:58,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/4fb4f22e9f2644378c2e4d7d273e5577, entries=150, sequenceid=518, filesize=12.0 K 2024-11-21T01:44:58,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/611c615ec5004ca48dc29bebf1dd428e as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/611c615ec5004ca48dc29bebf1dd428e 2024-11-21T01:44:58,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/611c615ec5004ca48dc29bebf1dd428e, entries=150, sequenceid=518, filesize=12.0 K 2024-11-21T01:44:58,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T01:44:58,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:58,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:44:58,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:58,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:58,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:58,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:58,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 70b50c956032fda6af9bbc7628072175 in 922ms, sequenceid=518, compaction requested=true 2024-11-21T01:44:58,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:58,784 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:58,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:44:58,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:58,784 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:58,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:44:58,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:58,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:44:58,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:58,787 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:58,787 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 70b50c956032fda6af9bbc7628072175/A is initiating minor compaction (all files) 2024-11-21T01:44:58,787 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70b50c956032fda6af9bbc7628072175/A in TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:58,787 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/e0dbfbe3180d484197bcfd7b8d19a05c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/aaa6db9d19874414a3dcd8001d264eee, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/46729d28357c42ba823d6d8356132dd5] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp, totalSize=36.6 K 2024-11-21T01:44:58,787 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:58,787 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 70b50c956032fda6af9bbc7628072175/B is initiating minor compaction (all files) 2024-11-21T01:44:58,787 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70b50c956032fda6af9bbc7628072175/B in TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:58,787 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/4295f090220140c29d189290084671ae, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/717b5232cc9b476b8710063459aca70e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/4fb4f22e9f2644378c2e4d7d273e5577] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp, totalSize=36.6 K 2024-11-21T01:44:58,788 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 4295f090220140c29d189290084671ae, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1732153496904 2024-11-21T01:44:58,788 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0dbfbe3180d484197bcfd7b8d19a05c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1732153496904 2024-11-21T01:44:58,788 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 717b5232cc9b476b8710063459aca70e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1732153496968 2024-11-21T01:44:58,789 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting aaa6db9d19874414a3dcd8001d264eee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1732153496968 2024-11-21T01:44:58,789 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fb4f22e9f2644378c2e4d7d273e5577, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1732153497722 2024-11-21T01:44:58,789 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46729d28357c42ba823d6d8356132dd5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1732153497722 2024-11-21T01:44:58,812 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70b50c956032fda6af9bbc7628072175#A#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:44:58,813 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/0d2e89cff2c1484fb113de30029ede9e is 50, key is test_row_0/A:col10/1732153497860/Put/seqid=0 2024-11-21T01:44:58,827 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70b50c956032fda6af9bbc7628072175#B#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:44:58,828 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/821212b4c0a6427c909e71e95d8b2fb0 is 50, key is test_row_0/B:col10/1732153497860/Put/seqid=0 2024-11-21T01:44:58,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742122_1298 (size=12949) 2024-11-21T01:44:58,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742121_1297 (size=12949) 2024-11-21T01:44:58,856 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/821212b4c0a6427c909e71e95d8b2fb0 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/821212b4c0a6427c909e71e95d8b2fb0 2024-11-21T01:44:58,861 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/0d2e89cff2c1484fb113de30029ede9e as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/0d2e89cff2c1484fb113de30029ede9e 2024-11-21T01:44:58,867 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70b50c956032fda6af9bbc7628072175/B of 70b50c956032fda6af9bbc7628072175 into 821212b4c0a6427c909e71e95d8b2fb0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:58,867 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:58,867 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., storeName=70b50c956032fda6af9bbc7628072175/B, priority=13, startTime=1732153498784; duration=0sec 2024-11-21T01:44:58,867 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:44:58,867 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:B 2024-11-21T01:44:58,867 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:44:58,871 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70b50c956032fda6af9bbc7628072175/A of 70b50c956032fda6af9bbc7628072175 into 0d2e89cff2c1484fb113de30029ede9e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:58,871 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:58,871 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., storeName=70b50c956032fda6af9bbc7628072175/A, priority=13, startTime=1732153498783; duration=0sec 2024-11-21T01:44:58,871 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:58,871 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:A 2024-11-21T01:44:58,871 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:44:58,871 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 70b50c956032fda6af9bbc7628072175/C is initiating minor compaction (all files) 2024-11-21T01:44:58,871 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70b50c956032fda6af9bbc7628072175/C in TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:58,871 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/2a6f9ae41de64a36bd2b63549a0ddd85, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/c4b010a4b9e643b1b0d7806538caa0f1, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/611c615ec5004ca48dc29bebf1dd428e] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp, totalSize=36.6 K 2024-11-21T01:44:58,872 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a6f9ae41de64a36bd2b63549a0ddd85, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1732153496904 2024-11-21T01:44:58,872 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting c4b010a4b9e643b1b0d7806538caa0f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1732153496968 2024-11-21T01:44:58,872 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 611c615ec5004ca48dc29bebf1dd428e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1732153497722 2024-11-21T01:44:58,897 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70b50c956032fda6af9bbc7628072175#C#compaction#254 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:44:58,898 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/d58844f988ab435ba06ee7facf203b27 is 50, key is test_row_0/C:col10/1732153497860/Put/seqid=0 2024-11-21T01:44:58,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742123_1299 (size=12949) 2024-11-21T01:44:58,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T01:44:58,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:58,934 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 70b50c956032fda6af9bbc7628072175 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-21T01:44:58,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=A 2024-11-21T01:44:58,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:58,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=B 2024-11-21T01:44:58,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:58,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=C 2024-11-21T01:44:58,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:58,938 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/d58844f988ab435ba06ee7facf203b27 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/d58844f988ab435ba06ee7facf203b27 2024-11-21T01:44:58,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T01:44:58,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/89a9b2e50c0543be85c651a47a198e8f is 50, key is test_row_0/A:col10/1732153497891/Put/seqid=0 2024-11-21T01:44:58,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742124_1300 (size=12301) 2024-11-21T01:44:58,952 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70b50c956032fda6af9bbc7628072175/C of 70b50c956032fda6af9bbc7628072175 into d58844f988ab435ba06ee7facf203b27(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:44:58,952 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:58,952 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., storeName=70b50c956032fda6af9bbc7628072175/C, priority=13, startTime=1732153498785; duration=0sec 2024-11-21T01:44:58,952 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:44:58,953 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:C 2024-11-21T01:44:58,954 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=536 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/89a9b2e50c0543be85c651a47a198e8f 2024-11-21T01:44:58,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/eb7dfd4899234b6d9747db6745380deb is 50, key is test_row_0/B:col10/1732153497891/Put/seqid=0 2024-11-21T01:44:58,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742125_1301 (size=12301) 2024-11-21T01:44:58,988 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=536 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/eb7dfd4899234b6d9747db6745380deb 2024-11-21T01:44:59,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/74eb05e4becf444b91332cde9a1b5bdb is 50, key is test_row_0/C:col10/1732153497891/Put/seqid=0 2024-11-21T01:44:59,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:44:59,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 70b50c956032fda6af9bbc7628072175 2024-11-21T01:44:59,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742126_1302 (size=12301) 2024-11-21T01:44:59,051 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=536 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/74eb05e4becf444b91332cde9a1b5bdb 2024-11-21T01:44:59,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/89a9b2e50c0543be85c651a47a198e8f as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/89a9b2e50c0543be85c651a47a198e8f 2024-11-21T01:44:59,071 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/89a9b2e50c0543be85c651a47a198e8f, entries=150, sequenceid=536, filesize=12.0 K 2024-11-21T01:44:59,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/eb7dfd4899234b6d9747db6745380deb as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/eb7dfd4899234b6d9747db6745380deb 2024-11-21T01:44:59,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:59,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153559063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:59,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:59,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:59,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153559063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:59,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:59,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:59,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153559064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:59,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:59,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:59,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153559067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:59,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:59,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153559069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:59,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:59,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:59,078 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/eb7dfd4899234b6d9747db6745380deb, entries=150, sequenceid=536, filesize=12.0 K 2024-11-21T01:44:59,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/74eb05e4becf444b91332cde9a1b5bdb as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/74eb05e4becf444b91332cde9a1b5bdb 2024-11-21T01:44:59,088 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/74eb05e4becf444b91332cde9a1b5bdb, entries=150, sequenceid=536, filesize=12.0 K 2024-11-21T01:44:59,090 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 70b50c956032fda6af9bbc7628072175 in 156ms, sequenceid=536, compaction requested=false 2024-11-21T01:44:59,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:44:59,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:59,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-21T01:44:59,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-21T01:44:59,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-21T01:44:59,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 465 msec 2024-11-21T01:44:59,098 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 472 msec 2024-11-21T01:44:59,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 70b50c956032fda6af9bbc7628072175 2024-11-21T01:44:59,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70b50c956032fda6af9bbc7628072175 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-21T01:44:59,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=A 2024-11-21T01:44:59,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:59,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=B 2024-11-21T01:44:59,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:59,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=C 2024-11-21T01:44:59,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:44:59,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/e6462b64ca9044a9bbce404baf5de3a5 is 50, key is test_row_0/A:col10/1732153499188/Put/seqid=0 2024-11-21T01:44:59,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:59,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153559207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:59,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:59,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153559210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:59,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:59,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:59,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153559212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:59,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:59,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153559213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:59,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153559214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:59,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742127_1303 (size=19621) 2024-11-21T01:44:59,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=557 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/e6462b64ca9044a9bbce404baf5de3a5 2024-11-21T01:44:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T01:44:59,253 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:44:59,254 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:44:59,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-21T01:44:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T01:44:59,263 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:44:59,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/fa1470e836f24a8aa6b551ff94b2a2eb is 50, key is test_row_0/B:col10/1732153499188/Put/seqid=0 2024-11-21T01:44:59,264 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:44:59,264 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:44:59,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742128_1304 (size=12301) 2024-11-21T01:44:59,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:59,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153559324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:59,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:59,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:59,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153559325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:59,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:59,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:59,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153559334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:59,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:59,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:59,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153559334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:59,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:59,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:59,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153559334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:59,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:59,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T01:44:59,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T01:44:59,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:59,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:44:59,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:59,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:59,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:59,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:59,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:59,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153559533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:59,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:59,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:59,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153559534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:59,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:59,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:59,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153559542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:59,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:59,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153559543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:59,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:59,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153559543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:59,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:59,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:59,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T01:44:59,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:59,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:44:59,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:59,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:59,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:59,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:59,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T01:44:59,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=557 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/fa1470e836f24a8aa6b551ff94b2a2eb 2024-11-21T01:44:59,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T01:44:59,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:59,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:44:59,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:59,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:59,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:59,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/a417be84c3e348259a62baece25260ef is 50, key is test_row_0/C:col10/1732153499188/Put/seqid=0 2024-11-21T01:44:59,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:59,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742129_1305 (size=12301) 2024-11-21T01:44:59,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:59,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153559843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:59,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:59,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153559843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:59,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:59,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:59,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:59,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153559853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:59,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:59,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:59,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153559853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:59,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:44:59,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153559853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:44:59,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:59,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:44:59,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:44:59,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T01:44:59,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T01:44:59,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:59,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:44:59,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:44:59,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:59,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:44:59,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:00,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T01:45:00,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:00,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:45:00,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:00,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:00,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:00,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:00,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T01:45:00,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:00,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:45:00,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:00,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:00,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:00,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:00,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=557 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/a417be84c3e348259a62baece25260ef 2024-11-21T01:45:00,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/e6462b64ca9044a9bbce404baf5de3a5 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/e6462b64ca9044a9bbce404baf5de3a5 2024-11-21T01:45:00,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/e6462b64ca9044a9bbce404baf5de3a5, entries=300, sequenceid=557, filesize=19.2 K 2024-11-21T01:45:00,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/fa1470e836f24a8aa6b551ff94b2a2eb as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/fa1470e836f24a8aa6b551ff94b2a2eb 2024-11-21T01:45:00,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/fa1470e836f24a8aa6b551ff94b2a2eb, entries=150, sequenceid=557, filesize=12.0 K 2024-11-21T01:45:00,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/a417be84c3e348259a62baece25260ef as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/a417be84c3e348259a62baece25260ef 2024-11-21T01:45:00,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/a417be84c3e348259a62baece25260ef, entries=150, sequenceid=557, filesize=12.0 K 2024-11-21T01:45:00,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 70b50c956032fda6af9bbc7628072175 in 1032ms, sequenceid=557, compaction requested=true 2024-11-21T01:45:00,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:45:00,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:45:00,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:00,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:45:00,220 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:00,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:00,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:45:00,221 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:00,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:00,222 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:00,222 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 70b50c956032fda6af9bbc7628072175/B is initiating minor compaction (all files) 2024-11-21T01:45:00,222 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70b50c956032fda6af9bbc7628072175/B in TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:00,222 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/821212b4c0a6427c909e71e95d8b2fb0, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/eb7dfd4899234b6d9747db6745380deb, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/fa1470e836f24a8aa6b551ff94b2a2eb] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp, totalSize=36.7 K 2024-11-21T01:45:00,222 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44871 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:00,222 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 70b50c956032fda6af9bbc7628072175/A is initiating minor compaction (all files) 2024-11-21T01:45:00,222 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70b50c956032fda6af9bbc7628072175/A in TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:00,222 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/0d2e89cff2c1484fb113de30029ede9e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/89a9b2e50c0543be85c651a47a198e8f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/e6462b64ca9044a9bbce404baf5de3a5] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp, totalSize=43.8 K 2024-11-21T01:45:00,223 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d2e89cff2c1484fb113de30029ede9e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1732153497722 2024-11-21T01:45:00,223 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 821212b4c0a6427c909e71e95d8b2fb0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1732153497722 2024-11-21T01:45:00,224 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting eb7dfd4899234b6d9747db6745380deb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=536, earliestPutTs=1732153497889 2024-11-21T01:45:00,224 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89a9b2e50c0543be85c651a47a198e8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=536, earliestPutTs=1732153497889 2024-11-21T01:45:00,224 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6462b64ca9044a9bbce404baf5de3a5, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=557, earliestPutTs=1732153499051 2024-11-21T01:45:00,224 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting fa1470e836f24a8aa6b551ff94b2a2eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=557, earliestPutTs=1732153499051 2024-11-21T01:45:00,235 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70b50c956032fda6af9bbc7628072175#A#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:00,236 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70b50c956032fda6af9bbc7628072175#B#compaction#262 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:00,236 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/1ee94be136c1452abc6d696e7604aef5 is 50, key is test_row_0/A:col10/1732153499188/Put/seqid=0 2024-11-21T01:45:00,236 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/afd258ce1875486c92e5716e42fccff5 is 50, key is test_row_0/B:col10/1732153499188/Put/seqid=0 2024-11-21T01:45:00,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742131_1307 (size=13051) 2024-11-21T01:45:00,258 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/afd258ce1875486c92e5716e42fccff5 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/afd258ce1875486c92e5716e42fccff5 2024-11-21T01:45:00,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742130_1306 (size=13051) 2024-11-21T01:45:00,267 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70b50c956032fda6af9bbc7628072175/B of 70b50c956032fda6af9bbc7628072175 into afd258ce1875486c92e5716e42fccff5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:00,267 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:45:00,268 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., storeName=70b50c956032fda6af9bbc7628072175/B, priority=13, startTime=1732153500220; duration=0sec 2024-11-21T01:45:00,269 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:00,269 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:B 2024-11-21T01:45:00,269 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:00,270 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/1ee94be136c1452abc6d696e7604aef5 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/1ee94be136c1452abc6d696e7604aef5 2024-11-21T01:45:00,271 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:00,271 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 70b50c956032fda6af9bbc7628072175/C is initiating minor compaction (all files) 2024-11-21T01:45:00,271 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70b50c956032fda6af9bbc7628072175/C in TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:00,271 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/d58844f988ab435ba06ee7facf203b27, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/74eb05e4becf444b91332cde9a1b5bdb, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/a417be84c3e348259a62baece25260ef] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp, totalSize=36.7 K 2024-11-21T01:45:00,272 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting d58844f988ab435ba06ee7facf203b27, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1732153497722 2024-11-21T01:45:00,272 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 74eb05e4becf444b91332cde9a1b5bdb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=536, earliestPutTs=1732153497889 2024-11-21T01:45:00,273 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting a417be84c3e348259a62baece25260ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=557, earliestPutTs=1732153499051 2024-11-21T01:45:00,276 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70b50c956032fda6af9bbc7628072175/A of 70b50c956032fda6af9bbc7628072175 into 1ee94be136c1452abc6d696e7604aef5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:00,277 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:45:00,277 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., storeName=70b50c956032fda6af9bbc7628072175/A, priority=13, startTime=1732153500220; duration=0sec 2024-11-21T01:45:00,277 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:00,277 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:A 2024-11-21T01:45:00,283 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70b50c956032fda6af9bbc7628072175#C#compaction#263 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:00,284 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/ddc05aa2fb4c490ba2848306131304cd is 50, key is test_row_0/C:col10/1732153499188/Put/seqid=0 2024-11-21T01:45:00,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742132_1308 (size=13051) 2024-11-21T01:45:00,306 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/ddc05aa2fb4c490ba2848306131304cd as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/ddc05aa2fb4c490ba2848306131304cd 2024-11-21T01:45:00,317 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70b50c956032fda6af9bbc7628072175/C of 70b50c956032fda6af9bbc7628072175 into ddc05aa2fb4c490ba2848306131304cd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:00,317 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:45:00,317 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., storeName=70b50c956032fda6af9bbc7628072175/C, priority=13, startTime=1732153500221; duration=0sec 2024-11-21T01:45:00,317 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:00,317 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:C 2024-11-21T01:45:00,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T01:45:00,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:00,346 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 70b50c956032fda6af9bbc7628072175 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T01:45:00,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=A 2024-11-21T01:45:00,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:00,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=B 2024-11-21T01:45:00,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:00,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=C 2024-11-21T01:45:00,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:00,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/8d353fa1104442feb043c23b711ca447 is 50, key is test_row_0/A:col10/1732153499209/Put/seqid=0 2024-11-21T01:45:00,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742133_1309 (size=12301) 2024-11-21T01:45:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 70b50c956032fda6af9bbc7628072175 2024-11-21T01:45:00,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:45:00,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T01:45:00,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:00,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153560419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:00,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:00,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:00,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153560419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:00,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153560419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:00,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:00,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:00,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:00,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:00,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:00,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:00,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:00,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:00,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:00,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153560425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:00,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:00,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:00,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:00,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153560427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:00,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:00,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:00,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:00,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:00,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153560532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:00,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:00,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153560533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:00,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:00,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153560534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:00,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:00,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:00,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:00,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:00,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153560543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:00,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:00,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:00,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:00,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153560543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:00,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:00,546 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:00,546 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:00,546 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:00,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153560745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:00,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:00,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:00,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:00,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:00,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153560746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:00,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:00,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:00,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:00,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:00,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153560749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:00,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:00,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:00,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:00,762 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/8d353fa1104442feb043c23b711ca447 2024-11-21T01:45:00,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:00,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153560758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:00,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:00,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153560758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:00,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:00,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:00,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:00,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/33ac50410a36439888b276f17f07c42c is 50, key is test_row_0/B:col10/1732153499209/Put/seqid=0 2024-11-21T01:45:00,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742134_1310 (size=12301) 2024-11-21T01:45:01,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:01,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153561052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:01,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:01,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:01,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:01,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153561053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:01,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:01,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:01,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:01,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:01,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:01,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153561063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:01,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:01,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:01,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:01,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:01,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153561073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:01,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:01,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:01,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:01,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153561073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:01,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:01,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:01,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:01,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:01,245 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/33ac50410a36439888b276f17f07c42c 2024-11-21T01:45:01,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/1a7ce430785d492b8b7164178ec17148 is 50, key is test_row_0/C:col10/1732153499209/Put/seqid=0 2024-11-21T01:45:01,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742135_1311 (size=12301) 2024-11-21T01:45:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T01:45:01,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:01,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153561562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:01,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:01,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:01,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:01,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153561563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:01,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:01,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:01,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:01,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:01,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153561577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:01,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:01,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:01,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:01,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:01,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153561583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:01,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:01,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153561583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:01,683 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/1a7ce430785d492b8b7164178ec17148 2024-11-21T01:45:01,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/8d353fa1104442feb043c23b711ca447 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/8d353fa1104442feb043c23b711ca447 2024-11-21T01:45:01,696 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/8d353fa1104442feb043c23b711ca447, entries=150, sequenceid=572, filesize=12.0 K 2024-11-21T01:45:01,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/33ac50410a36439888b276f17f07c42c as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/33ac50410a36439888b276f17f07c42c 2024-11-21T01:45:01,703 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/33ac50410a36439888b276f17f07c42c, entries=150, sequenceid=572, filesize=12.0 K 2024-11-21T01:45:01,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/1a7ce430785d492b8b7164178ec17148 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/1a7ce430785d492b8b7164178ec17148 2024-11-21T01:45:01,708 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/1a7ce430785d492b8b7164178ec17148, entries=150, sequenceid=572, filesize=12.0 K 2024-11-21T01:45:01,709 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 70b50c956032fda6af9bbc7628072175 in 1363ms, sequenceid=572, compaction requested=false 2024-11-21T01:45:01,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:45:01,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:01,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-21T01:45:01,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-21T01:45:01,713 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-21T01:45:01,713 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4470 sec 2024-11-21T01:45:01,716 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.4600 sec 2024-11-21T01:45:02,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 70b50c956032fda6af9bbc7628072175 2024-11-21T01:45:02,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70b50c956032fda6af9bbc7628072175 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-21T01:45:02,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=A 2024-11-21T01:45:02,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:45:02,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=B 2024-11-21T01:45:02,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:45:02,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=C 2024-11-21T01:45:02,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:45:02,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/0158e6c8dcdb47159d04b0f7ea2a8187 is 50, key is test_row_0/A:col10/1732153500423/Put/seqid=0 2024-11-21T01:45:02,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742136_1312 (size=12301) 2024-11-21T01:45:02,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=593 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/0158e6c8dcdb47159d04b0f7ea2a8187 2024-11-21T01:45:02,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:02,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153562600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:02,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:02,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153562600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:02,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:02,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:02,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:02,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153562602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:02,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:02,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:02,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:02,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153562603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:02,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:02,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:02,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:02,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:02,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:02,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:02,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153562603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:02,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:02,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:02,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:02,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/505e70b1c7864ebcab84619e706c22c3 is 50, key is test_row_0/B:col10/1732153500423/Put/seqid=0 2024-11-21T01:45:02,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742137_1313 (size=12301) 2024-11-21T01:45:02,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:02,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153562713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:02,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:02,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:02,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153562713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:02,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:02,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:02,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:02,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:02,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153562714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:02,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:02,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:02,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:02,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153562714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:02,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:02,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:02,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:02,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:02,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:02,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:02,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:02,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153562714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:02,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:02,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:02,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:02,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:02,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153562923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:02,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:02,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153562923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:02,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:02,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153562923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:02,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:02,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:02,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153562923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:02,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:02,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:02,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:02,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:02,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153562924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:02,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:02,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:02,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:02,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:02,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:03,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=593 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/505e70b1c7864ebcab84619e706c22c3 2024-11-21T01:45:03,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/ea8b86b0e37c4f09a08c41b80f968035 is 50, key is test_row_0/C:col10/1732153500423/Put/seqid=0 2024-11-21T01:45:03,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742138_1314 (size=12301) 2024-11-21T01:45:03,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:03,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153563233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:03,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:03,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153563233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:03,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:03,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:03,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:03,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:03,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:03,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153563233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:03,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:03,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:03,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:03,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:03,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153563233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:03,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:03,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:03,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:03,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:03,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153563233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:03,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:03,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:03,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T01:45:03,413 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:45:03,413 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:45:03,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-21T01:45:03,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T01:45:03,415 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:45:03,416 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:45:03,416 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:45:03,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=593 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/ea8b86b0e37c4f09a08c41b80f968035 2024-11-21T01:45:03,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/0158e6c8dcdb47159d04b0f7ea2a8187 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/0158e6c8dcdb47159d04b0f7ea2a8187 2024-11-21T01:45:03,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/0158e6c8dcdb47159d04b0f7ea2a8187, entries=150, sequenceid=593, filesize=12.0 K 2024-11-21T01:45:03,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/505e70b1c7864ebcab84619e706c22c3 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/505e70b1c7864ebcab84619e706c22c3 2024-11-21T01:45:03,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/505e70b1c7864ebcab84619e706c22c3, entries=150, sequenceid=593, filesize=12.0 K 2024-11-21T01:45:03,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/ea8b86b0e37c4f09a08c41b80f968035 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/ea8b86b0e37c4f09a08c41b80f968035 2024-11-21T01:45:03,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/ea8b86b0e37c4f09a08c41b80f968035, entries=150, sequenceid=593, filesize=12.0 K 2024-11-21T01:45:03,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 70b50c956032fda6af9bbc7628072175 in 914ms, sequenceid=593, compaction requested=true 2024-11-21T01:45:03,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:45:03,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:45:03,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:03,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:45:03,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:03,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:45:03,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:03,491 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:03,491 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:03,492 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:03,492 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 70b50c956032fda6af9bbc7628072175/B is initiating minor compaction (all files) 2024-11-21T01:45:03,492 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70b50c956032fda6af9bbc7628072175/B in TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:03,492 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/afd258ce1875486c92e5716e42fccff5, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/33ac50410a36439888b276f17f07c42c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/505e70b1c7864ebcab84619e706c22c3] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp, totalSize=36.8 K 2024-11-21T01:45:03,492 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:03,492 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 70b50c956032fda6af9bbc7628072175/A is initiating minor compaction (all files) 2024-11-21T01:45:03,492 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70b50c956032fda6af9bbc7628072175/A in TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:03,493 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/1ee94be136c1452abc6d696e7604aef5, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/8d353fa1104442feb043c23b711ca447, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/0158e6c8dcdb47159d04b0f7ea2a8187] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp, totalSize=36.8 K 2024-11-21T01:45:03,493 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting afd258ce1875486c92e5716e42fccff5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=557, earliestPutTs=1732153499051 2024-11-21T01:45:03,493 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ee94be136c1452abc6d696e7604aef5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=557, earliestPutTs=1732153499051 2024-11-21T01:45:03,494 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d353fa1104442feb043c23b711ca447, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1732153499208 2024-11-21T01:45:03,494 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 33ac50410a36439888b276f17f07c42c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1732153499208 2024-11-21T01:45:03,494 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0158e6c8dcdb47159d04b0f7ea2a8187, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=593, earliestPutTs=1732153500423 2024-11-21T01:45:03,494 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 505e70b1c7864ebcab84619e706c22c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=593, earliestPutTs=1732153500423 2024-11-21T01:45:03,504 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70b50c956032fda6af9bbc7628072175#B#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:03,505 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/aeb501fc178846aa831ab29bae111ca9 is 50, key is test_row_0/B:col10/1732153500423/Put/seqid=0 2024-11-21T01:45:03,511 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70b50c956032fda6af9bbc7628072175#A#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:03,511 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/c27a2f3be6864544bf18fb2e0d0c477f is 50, key is test_row_0/A:col10/1732153500423/Put/seqid=0 2024-11-21T01:45:03,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742139_1315 (size=13153) 2024-11-21T01:45:03,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T01:45:03,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742140_1316 (size=13153) 2024-11-21T01:45:03,540 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/c27a2f3be6864544bf18fb2e0d0c477f as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/c27a2f3be6864544bf18fb2e0d0c477f 2024-11-21T01:45:03,553 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70b50c956032fda6af9bbc7628072175/A of 70b50c956032fda6af9bbc7628072175 into c27a2f3be6864544bf18fb2e0d0c477f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:03,553 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:45:03,553 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., storeName=70b50c956032fda6af9bbc7628072175/A, priority=13, startTime=1732153503491; duration=0sec 2024-11-21T01:45:03,553 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:03,553 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:A 2024-11-21T01:45:03,553 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:03,554 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:03,554 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 70b50c956032fda6af9bbc7628072175/C is initiating minor compaction (all files) 2024-11-21T01:45:03,554 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70b50c956032fda6af9bbc7628072175/C in TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:03,555 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/ddc05aa2fb4c490ba2848306131304cd, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/1a7ce430785d492b8b7164178ec17148, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/ea8b86b0e37c4f09a08c41b80f968035] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp, totalSize=36.8 K 2024-11-21T01:45:03,556 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddc05aa2fb4c490ba2848306131304cd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=557, earliestPutTs=1732153499051 2024-11-21T01:45:03,556 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a7ce430785d492b8b7164178ec17148, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1732153499208 2024-11-21T01:45:03,556 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea8b86b0e37c4f09a08c41b80f968035, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=593, earliestPutTs=1732153500423 2024-11-21T01:45:03,562 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70b50c956032fda6af9bbc7628072175#C#compaction#272 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:03,563 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/cbacd76999af4f5abb916221139c4a40 is 50, key is test_row_0/C:col10/1732153500423/Put/seqid=0 2024-11-21T01:45:03,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742141_1317 (size=13153) 2024-11-21T01:45:03,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T01:45:03,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:03,569 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 70b50c956032fda6af9bbc7628072175 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-21T01:45:03,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=A 2024-11-21T01:45:03,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:03,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=B 2024-11-21T01:45:03,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:03,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=C 2024-11-21T01:45:03,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:03,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/c147c1af44184fdf8fd84481efd47a6e is 50, key is test_row_0/A:col10/1732153502599/Put/seqid=0 2024-11-21T01:45:03,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742142_1318 (size=12301) 2024-11-21T01:45:03,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T01:45:03,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 70b50c956032fda6af9bbc7628072175 2024-11-21T01:45:03,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:45:03,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:03,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153563770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:03,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153563772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:03,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153563772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:03,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732153503760, max timestamp=1732153503775Num uniques 859; 2024-11-21T01:45:03,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732153503760, max timestamp=1732153503775Num uniques 859; 2024-11-21T01:45:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:03,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T01:45:03,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T01:45:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:03,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732153503760, max timestamp=1732153503775Num uniques 859; 2024-11-21T01:45:03,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T01:45:03,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153563828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:03,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:03,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:03,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:03,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:03,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153563831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:03,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:03,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:03,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:03,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:03,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153563883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:03,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:03,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153563883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:03,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:03,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:03,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:03,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:03,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:03,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:03,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:03,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153563883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:03,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:03,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:03,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:03,931 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/aeb501fc178846aa831ab29bae111ca9 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/aeb501fc178846aa831ab29bae111ca9 2024-11-21T01:45:03,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:03,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153563945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:03,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:03,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:03,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:03,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:03,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153563945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:03,951 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70b50c956032fda6af9bbc7628072175/B of 70b50c956032fda6af9bbc7628072175 into aeb501fc178846aa831ab29bae111ca9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:03,951 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:45:03,951 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., storeName=70b50c956032fda6af9bbc7628072175/B, priority=13, startTime=1732153503491; duration=0sec 2024-11-21T01:45:03,951 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:03,951 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:B 2024-11-21T01:45:03,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:03,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:03,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:03,974 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/cbacd76999af4f5abb916221139c4a40 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/cbacd76999af4f5abb916221139c4a40 2024-11-21T01:45:03,980 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70b50c956032fda6af9bbc7628072175/C of 70b50c956032fda6af9bbc7628072175 into cbacd76999af4f5abb916221139c4a40(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:03,980 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:45:03,980 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., storeName=70b50c956032fda6af9bbc7628072175/C, priority=13, startTime=1732153503491; duration=0sec 2024-11-21T01:45:03,980 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:03,980 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:C 2024-11-21T01:45:03,980 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=605 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/c147c1af44184fdf8fd84481efd47a6e 2024-11-21T01:45:03,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/e1f0f02eb473460f9b10c6e73d77f336 is 50, key is test_row_0/B:col10/1732153502599/Put/seqid=0 2024-11-21T01:45:04,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742143_1319 (size=12301) 2024-11-21T01:45:04,014 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=605 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/e1f0f02eb473460f9b10c6e73d77f336 2024-11-21T01:45:04,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/5680889a2a9e4ca5828b800aa7b2860a is 50, key is test_row_0/C:col10/1732153502599/Put/seqid=0 2024-11-21T01:45:04,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T01:45:04,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742144_1320 (size=12301) 2024-11-21T01:45:04,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:04,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153564093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:04,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:04,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:04,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:04,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:04,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153564093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:04,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:04,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:04,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:04,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:04,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153564106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:04,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:04,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:04,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:04,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:04,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153564153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:04,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:04,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:04,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:04,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:04,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153564165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:04,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:04,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:04,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:04,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:04,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153564403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:04,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:04,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153564404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:04,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:04,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:04,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:04,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:04,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:04,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:04,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:04,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153564424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:04,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:04,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:04,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:04,460 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=605 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/5680889a2a9e4ca5828b800aa7b2860a 2024-11-21T01:45:04,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:04,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153564462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:04,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:04,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:04,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:04,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/c147c1af44184fdf8fd84481efd47a6e as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/c147c1af44184fdf8fd84481efd47a6e 2024-11-21T01:45:04,473 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/c147c1af44184fdf8fd84481efd47a6e, entries=150, sequenceid=605, filesize=12.0 K 2024-11-21T01:45:04,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/e1f0f02eb473460f9b10c6e73d77f336 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/e1f0f02eb473460f9b10c6e73d77f336 2024-11-21T01:45:04,478 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/e1f0f02eb473460f9b10c6e73d77f336, entries=150, sequenceid=605, filesize=12.0 K 2024-11-21T01:45:04,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/5680889a2a9e4ca5828b800aa7b2860a as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/5680889a2a9e4ca5828b800aa7b2860a 2024-11-21T01:45:04,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:04,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153564473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:04,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:04,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:04,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:04,486 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/5680889a2a9e4ca5828b800aa7b2860a, entries=150, sequenceid=605, filesize=12.0 K 2024-11-21T01:45:04,487 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 70b50c956032fda6af9bbc7628072175 in 918ms, sequenceid=605, compaction requested=false 2024-11-21T01:45:04,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:45:04,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:04,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-21T01:45:04,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-21T01:45:04,495 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-21T01:45:04,496 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0770 sec 2024-11-21T01:45:04,500 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.0850 sec 2024-11-21T01:45:04,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T01:45:04,553 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:45:04,554 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:45:04,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-21T01:45:04,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T01:45:04,556 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:45:04,557 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:45:04,558 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:45:04,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T01:45:04,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-21T01:45:04,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:04,712 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 70b50c956032fda6af9bbc7628072175 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-21T01:45:04,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=A 2024-11-21T01:45:04,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:45:04,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=B 2024-11-21T01:45:04,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:45:04,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=C 2024-11-21T01:45:04,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:45:04,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/b58f0673fb3546839dd12b1309b59217 is 50, key is test_row_0/A:col10/1732153503799/Put/seqid=0 2024-11-21T01:45:04,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742145_1321 (size=12301) 2024-11-21T01:45:04,727 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=643 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/b58f0673fb3546839dd12b1309b59217 2024-11-21T01:45:04,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/a1e7493a0cf145a59e9ffc344a751d4f is 50, key is test_row_0/B:col10/1732153503799/Put/seqid=0 2024-11-21T01:45:04,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742146_1322 (size=12301) 2024-11-21T01:45:04,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T01:45:04,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 70b50c956032fda6af9bbc7628072175 2024-11-21T01:45:04,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:45:04,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:04,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153564983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:04,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:04,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153564989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:04,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:04,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153564990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:04,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:04,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153564991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:04,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:04,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153564993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:05,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:05,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153565102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:05,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:05,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153565103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:05,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:05,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:05,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153565103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:05,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:05,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:05,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153565103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:05,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:05,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:05,162 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=643 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/a1e7493a0cf145a59e9ffc344a751d4f 2024-11-21T01:45:05,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T01:45:05,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/c6f53093e29e41c0b8973c7564d460fb is 50, key is test_row_0/C:col10/1732153503799/Put/seqid=0 2024-11-21T01:45:05,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742147_1323 (size=12301) 2024-11-21T01:45:05,207 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=643 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/c6f53093e29e41c0b8973c7564d460fb 2024-11-21T01:45:05,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/b58f0673fb3546839dd12b1309b59217 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/b58f0673fb3546839dd12b1309b59217 2024-11-21T01:45:05,224 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/b58f0673fb3546839dd12b1309b59217, entries=150, sequenceid=643, filesize=12.0 K 2024-11-21T01:45:05,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/a1e7493a0cf145a59e9ffc344a751d4f as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/a1e7493a0cf145a59e9ffc344a751d4f 2024-11-21T01:45:05,235 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/a1e7493a0cf145a59e9ffc344a751d4f, entries=150, sequenceid=643, filesize=12.0 K 2024-11-21T01:45:05,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/c6f53093e29e41c0b8973c7564d460fb as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/c6f53093e29e41c0b8973c7564d460fb 2024-11-21T01:45:05,249 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/c6f53093e29e41c0b8973c7564d460fb, entries=150, sequenceid=643, filesize=12.0 K 2024-11-21T01:45:05,250 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 70b50c956032fda6af9bbc7628072175 in 538ms, sequenceid=643, compaction requested=true 2024-11-21T01:45:05,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:45:05,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:05,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-21T01:45:05,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-21T01:45:05,254 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-21T01:45:05,254 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 694 msec 2024-11-21T01:45:05,256 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 701 msec 2024-11-21T01:45:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 70b50c956032fda6af9bbc7628072175 2024-11-21T01:45:05,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70b50c956032fda6af9bbc7628072175 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T01:45:05,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=A 2024-11-21T01:45:05,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:05,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=B 2024-11-21T01:45:05,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:05,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=C 2024-11-21T01:45:05,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:05,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/b778c726a9d34d95a37cce35928618ad is 50, key is test_row_0/A:col10/1732153504989/Put/seqid=0 2024-11-21T01:45:05,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742148_1324 (size=12301) 2024-11-21T01:45:05,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153565340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:05,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:05,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:05,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153565343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:05,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:05,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153565343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:05,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:05,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153565353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:05,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:05,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:05,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153565452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:05,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:05,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:05,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153565452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:05,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:05,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:05,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153565463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:05,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:05,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153565463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:05,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:05,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:05,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:05,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153565663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:05,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153565663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:05,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:05,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153565673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:05,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:05,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153565673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:05,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:05,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T01:45:05,693 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:45:05,693 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:45:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-21T01:45:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T01:45:05,695 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:45:05,695 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:45:05,695 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:45:05,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=657 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/b778c726a9d34d95a37cce35928618ad 2024-11-21T01:45:05,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/27b46b4d33b24243be8ad7f32b0d0b0e is 50, key is test_row_0/B:col10/1732153504989/Put/seqid=0 2024-11-21T01:45:05,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742149_1325 (size=12301) 2024-11-21T01:45:05,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T01:45:05,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T01:45:05,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:05,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:45:05,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:05,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:05,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:05,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:05,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153565972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:05,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153565972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:05,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153565983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153565983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:06,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T01:45:06,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:06,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:45:06,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:06,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:06,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T01:45:06,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153566012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:06,015 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to b93ada835ef8,41129,1732153434359 for 'test_row_1' in 70b50c956032fda6af9bbc7628072175 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T01:45:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:06,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:06,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:06,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=657 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/27b46b4d33b24243be8ad7f32b0d0b0e 2024-11-21T01:45:06,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/e432aabf0f5344e6b91406e1ded2c765 is 50, key is test_row_0/C:col10/1732153504989/Put/seqid=0 2024-11-21T01:45:06,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742150_1326 (size=12301) 2024-11-21T01:45:06,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T01:45:06,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:06,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:45:06,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:06,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:06,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:06,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T01:45:06,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:06,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:45:06,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:06,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:06,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T01:45:06,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T01:45:06,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:06,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:45:06,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:06,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:06,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:06,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:06,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153566483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:06,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:06,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:06,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:06,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:06,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153566493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:06,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:06,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:06,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:06,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:06,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153566493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:06,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:06,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153566493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:06,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:06,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:06,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:06,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:06,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:06,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:06,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=657 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/e432aabf0f5344e6b91406e1ded2c765 2024-11-21T01:45:06,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/b778c726a9d34d95a37cce35928618ad as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/b778c726a9d34d95a37cce35928618ad 2024-11-21T01:45:06,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/b778c726a9d34d95a37cce35928618ad, entries=150, sequenceid=657, filesize=12.0 K 2024-11-21T01:45:06,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/27b46b4d33b24243be8ad7f32b0d0b0e as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/27b46b4d33b24243be8ad7f32b0d0b0e 2024-11-21T01:45:06,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/27b46b4d33b24243be8ad7f32b0d0b0e, entries=150, sequenceid=657, filesize=12.0 K 2024-11-21T01:45:06,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/e432aabf0f5344e6b91406e1ded2c765 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/e432aabf0f5344e6b91406e1ded2c765 2024-11-21T01:45:06,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/e432aabf0f5344e6b91406e1ded2c765, entries=150, sequenceid=657, filesize=12.0 K 2024-11-21T01:45:06,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 70b50c956032fda6af9bbc7628072175 in 1253ms, sequenceid=657, compaction requested=true 2024-11-21T01:45:06,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:45:06,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:45:06,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:06,570 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:45:06,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:45:06,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:06,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:45:06,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:06,570 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:45:06,571 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:45:06,571 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 70b50c956032fda6af9bbc7628072175/A is initiating minor compaction (all files) 2024-11-21T01:45:06,571 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70b50c956032fda6af9bbc7628072175/A in TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:06,571 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:45:06,571 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/c27a2f3be6864544bf18fb2e0d0c477f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/c147c1af44184fdf8fd84481efd47a6e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/b58f0673fb3546839dd12b1309b59217, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/b778c726a9d34d95a37cce35928618ad] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp, totalSize=48.9 K 2024-11-21T01:45:06,571 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 70b50c956032fda6af9bbc7628072175/B is initiating minor compaction (all files) 2024-11-21T01:45:06,571 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70b50c956032fda6af9bbc7628072175/B in TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:06,571 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/aeb501fc178846aa831ab29bae111ca9, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/e1f0f02eb473460f9b10c6e73d77f336, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/a1e7493a0cf145a59e9ffc344a751d4f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/27b46b4d33b24243be8ad7f32b0d0b0e] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp, totalSize=48.9 K 2024-11-21T01:45:06,572 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting c27a2f3be6864544bf18fb2e0d0c477f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=593, earliestPutTs=1732153500423 2024-11-21T01:45:06,572 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting aeb501fc178846aa831ab29bae111ca9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=593, earliestPutTs=1732153500423 2024-11-21T01:45:06,572 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting c147c1af44184fdf8fd84481efd47a6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=605, earliestPutTs=1732153502598 2024-11-21T01:45:06,572 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting e1f0f02eb473460f9b10c6e73d77f336, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=605, earliestPutTs=1732153502598 2024-11-21T01:45:06,572 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting b58f0673fb3546839dd12b1309b59217, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=643, earliestPutTs=1732153503799 2024-11-21T01:45:06,573 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting a1e7493a0cf145a59e9ffc344a751d4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=643, earliestPutTs=1732153503799 2024-11-21T01:45:06,573 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting b778c726a9d34d95a37cce35928618ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=657, earliestPutTs=1732153504989 2024-11-21T01:45:06,573 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 27b46b4d33b24243be8ad7f32b0d0b0e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=657, earliestPutTs=1732153504989 2024-11-21T01:45:06,583 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70b50c956032fda6af9bbc7628072175#A#compaction#282 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:06,584 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/91cc41fcd7c040aabf00778d0190e50a is 50, key is test_row_0/A:col10/1732153504989/Put/seqid=0 2024-11-21T01:45:06,584 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70b50c956032fda6af9bbc7628072175#B#compaction#283 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:06,585 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/084f6609c933420db154e82695f3900e is 50, key is test_row_0/B:col10/1732153504989/Put/seqid=0 2024-11-21T01:45:06,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742151_1327 (size=13289) 2024-11-21T01:45:06,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742152_1328 (size=13289) 2024-11-21T01:45:06,600 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/084f6609c933420db154e82695f3900e as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/084f6609c933420db154e82695f3900e 2024-11-21T01:45:06,604 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 70b50c956032fda6af9bbc7628072175/B of 70b50c956032fda6af9bbc7628072175 into 084f6609c933420db154e82695f3900e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:06,604 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:45:06,604 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., storeName=70b50c956032fda6af9bbc7628072175/B, priority=12, startTime=1732153506570; duration=0sec 2024-11-21T01:45:06,604 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:06,604 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:B 2024-11-21T01:45:06,605 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:45:06,606 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:45:06,606 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 70b50c956032fda6af9bbc7628072175/C is initiating minor compaction (all files) 2024-11-21T01:45:06,606 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70b50c956032fda6af9bbc7628072175/C in TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:06,606 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/cbacd76999af4f5abb916221139c4a40, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/5680889a2a9e4ca5828b800aa7b2860a, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/c6f53093e29e41c0b8973c7564d460fb, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/e432aabf0f5344e6b91406e1ded2c765] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp, totalSize=48.9 K 2024-11-21T01:45:06,606 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting cbacd76999af4f5abb916221139c4a40, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=593, earliestPutTs=1732153500423 2024-11-21T01:45:06,607 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 5680889a2a9e4ca5828b800aa7b2860a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=605, earliestPutTs=1732153502598 2024-11-21T01:45:06,608 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting c6f53093e29e41c0b8973c7564d460fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=643, earliestPutTs=1732153503799 2024-11-21T01:45:06,608 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting e432aabf0f5344e6b91406e1ded2c765, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=657, earliestPutTs=1732153504989 2024-11-21T01:45:06,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T01:45:06,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:06,613 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 70b50c956032fda6af9bbc7628072175 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-21T01:45:06,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=A 2024-11-21T01:45:06,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:06,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=B 2024-11-21T01:45:06,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:06,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=C 2024-11-21T01:45:06,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:06,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/ab749c8e55b3438c961afa633e4dc854 is 50, key is test_row_0/A:col10/1732153505341/Put/seqid=0 2024-11-21T01:45:06,626 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70b50c956032fda6af9bbc7628072175#C#compaction#285 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:06,626 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/aac2f6a8c5314dfc9241a911fc608f14 is 50, key is test_row_0/C:col10/1732153504989/Put/seqid=0 2024-11-21T01:45:06,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742153_1329 (size=12301) 2024-11-21T01:45:06,636 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=676 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/ab749c8e55b3438c961afa633e4dc854 2024-11-21T01:45:06,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742154_1330 (size=13289) 2024-11-21T01:45:06,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/4f581e9ae9374d78b05b0a504c1790b1 is 50, key is test_row_0/B:col10/1732153505341/Put/seqid=0 2024-11-21T01:45:06,648 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/aac2f6a8c5314dfc9241a911fc608f14 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/aac2f6a8c5314dfc9241a911fc608f14 2024-11-21T01:45:06,655 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 70b50c956032fda6af9bbc7628072175/C of 70b50c956032fda6af9bbc7628072175 into aac2f6a8c5314dfc9241a911fc608f14(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:06,655 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:45:06,655 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., storeName=70b50c956032fda6af9bbc7628072175/C, priority=12, startTime=1732153506570; duration=0sec 2024-11-21T01:45:06,655 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:06,655 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:C 2024-11-21T01:45:06,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742155_1331 (size=12301) 2024-11-21T01:45:06,658 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=676 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/4f581e9ae9374d78b05b0a504c1790b1 2024-11-21T01:45:06,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/8effbce987dd4fc6ad426434ca6d2d71 is 50, key is test_row_0/C:col10/1732153505341/Put/seqid=0 2024-11-21T01:45:06,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742156_1332 (size=12301) 2024-11-21T01:45:06,693 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=676 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/8effbce987dd4fc6ad426434ca6d2d71 2024-11-21T01:45:06,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/ab749c8e55b3438c961afa633e4dc854 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/ab749c8e55b3438c961afa633e4dc854 2024-11-21T01:45:06,703 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/ab749c8e55b3438c961afa633e4dc854, entries=150, sequenceid=676, filesize=12.0 K 2024-11-21T01:45:06,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/4f581e9ae9374d78b05b0a504c1790b1 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/4f581e9ae9374d78b05b0a504c1790b1 2024-11-21T01:45:06,707 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/4f581e9ae9374d78b05b0a504c1790b1, entries=150, sequenceid=676, filesize=12.0 K 2024-11-21T01:45:06,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/8effbce987dd4fc6ad426434ca6d2d71 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/8effbce987dd4fc6ad426434ca6d2d71 2024-11-21T01:45:06,713 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/8effbce987dd4fc6ad426434ca6d2d71, entries=150, sequenceid=676, filesize=12.0 K 2024-11-21T01:45:06,714 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 70b50c956032fda6af9bbc7628072175 in 101ms, sequenceid=676, compaction requested=false 2024-11-21T01:45:06,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:45:06,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:06,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-21T01:45:06,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-21T01:45:06,718 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-21T01:45:06,718 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0200 sec 2024-11-21T01:45:06,724 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.0260 sec 2024-11-21T01:45:06,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T01:45:06,833 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:45:06,833 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:45:06,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-21T01:45:06,836 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:45:06,837 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:45:06,837 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:45:06,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-21T01:45:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-21T01:45:06,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-21T01:45:06,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:06,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:45:06,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:06,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-21T01:45:06,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-21T01:45:06,996 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-21T01:45:06,997 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 156 msec 2024-11-21T01:45:06,999 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 163 msec 2024-11-21T01:45:07,004 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/91cc41fcd7c040aabf00778d0190e50a as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/91cc41fcd7c040aabf00778d0190e50a 2024-11-21T01:45:07,009 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 70b50c956032fda6af9bbc7628072175/A of 70b50c956032fda6af9bbc7628072175 into 91cc41fcd7c040aabf00778d0190e50a(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:07,009 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:45:07,009 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., storeName=70b50c956032fda6af9bbc7628072175/A, priority=12, startTime=1732153506570; duration=0sec 2024-11-21T01:45:07,010 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:07,010 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:A 2024-11-21T01:45:07,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-21T01:45:07,153 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:45:07,153 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:45:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-21T01:45:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-21T01:45:07,156 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:45:07,157 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:45:07,157 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:45:07,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-21T01:45:07,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-21T01:45:07,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:07,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:45:07,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:07,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-21T01:45:07,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-21T01:45:07,317 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-21T01:45:07,317 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 157 msec 2024-11-21T01:45:07,319 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 165 msec 2024-11-21T01:45:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-21T01:45:07,473 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:45:07,474 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:45:07,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees 2024-11-21T01:45:07,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-21T01:45:07,476 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:45:07,478 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:45:07,478 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:45:07,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 70b50c956032fda6af9bbc7628072175 2024-11-21T01:45:07,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70b50c956032fda6af9bbc7628072175 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-21T01:45:07,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=A 2024-11-21T01:45:07,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:07,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=B 2024-11-21T01:45:07,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:07,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=C 2024-11-21T01:45:07,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:07,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/0ef3e606179f478685c59d09d1fe4740 is 50, key is test_row_0/A:col10/1732153507505/Put/seqid=0 2024-11-21T01:45:07,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742157_1333 (size=12301) 2024-11-21T01:45:07,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:07,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153567541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:07,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:07,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153567542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:07,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:07,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153567543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:07,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:07,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153567545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:07,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:07,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:07,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:07,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-21T01:45:07,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-21T01:45:07,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:07,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:45:07,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:07,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:07,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:07,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:07,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:07,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153567653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:07,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:07,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:07,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:07,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153567653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:07,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:07,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:07,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:07,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:07,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:07,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153567653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:07,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:07,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:07,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:07,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:07,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153567653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:07,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:07,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:07,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:07,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-21T01:45:07,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:07,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:45:07,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:07,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:07,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:07,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:07,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-21T01:45:07,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:07,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153567863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:07,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:07,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153567863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:07,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:07,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:07,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:07,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:07,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:07,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:07,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153567863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:07,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:07,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:07,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:07,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153567864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:07,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:07,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:07,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=692 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/0ef3e606179f478685c59d09d1fe4740 2024-11-21T01:45:07,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/c5f809db18ab4bf1882ae32b1b84c940 is 50, key is test_row_0/B:col10/1732153507505/Put/seqid=0 2024-11-21T01:45:07,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-21T01:45:07,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:07,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:45:07,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:07,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:07,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:07,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:07,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742158_1334 (size=12301) 2024-11-21T01:45:07,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=692 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/c5f809db18ab4bf1882ae32b1b84c940 2024-11-21T01:45:07,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/91d04b169e364961b82d3b7a07331d74 is 50, key is test_row_0/C:col10/1732153507505/Put/seqid=0 2024-11-21T01:45:07,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742159_1335 (size=12301) 2024-11-21T01:45:08,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:08,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153568033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:08,037 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to b93ada835ef8,41129,1732153434359 for 'test_row_1' in 70b50c956032fda6af9bbc7628072175 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T01:45:08,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:08,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:08,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:08,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-21T01:45:08,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:08,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:45:08,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:08,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:08,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:08,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:08,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-21T01:45:08,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:08,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153568173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:08,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:08,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153568173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:08,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:08,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:08,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153568173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:08,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:08,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:08,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:08,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:08,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153568173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:08,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:08,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:08,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:08,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:08,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:08,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:08,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:08,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:08,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-21T01:45:08,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:08,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:45:08,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:08,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:08,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:08,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:08,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=692 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/91d04b169e364961b82d3b7a07331d74 2024-11-21T01:45:08,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/0ef3e606179f478685c59d09d1fe4740 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/0ef3e606179f478685c59d09d1fe4740 2024-11-21T01:45:08,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/0ef3e606179f478685c59d09d1fe4740, entries=150, sequenceid=692, filesize=12.0 K 2024-11-21T01:45:08,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/c5f809db18ab4bf1882ae32b1b84c940 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/c5f809db18ab4bf1882ae32b1b84c940 2024-11-21T01:45:08,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/c5f809db18ab4bf1882ae32b1b84c940, entries=150, sequenceid=692, filesize=12.0 K 2024-11-21T01:45:08,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/91d04b169e364961b82d3b7a07331d74 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/91d04b169e364961b82d3b7a07331d74 2024-11-21T01:45:08,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/91d04b169e364961b82d3b7a07331d74, entries=150, sequenceid=692, filesize=12.0 K 2024-11-21T01:45:08,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 70b50c956032fda6af9bbc7628072175 in 884ms, sequenceid=692, compaction requested=true 2024-11-21T01:45:08,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:45:08,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:45:08,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:08,399 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:08,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:45:08,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:08,399 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:08,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:45:08,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:08,400 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:08,400 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 70b50c956032fda6af9bbc7628072175/A is initiating minor compaction (all files) 2024-11-21T01:45:08,401 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70b50c956032fda6af9bbc7628072175/A in TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:08,401 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/91cc41fcd7c040aabf00778d0190e50a, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/ab749c8e55b3438c961afa633e4dc854, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/0ef3e606179f478685c59d09d1fe4740] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp, totalSize=37.0 K 2024-11-21T01:45:08,404 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:08,404 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 70b50c956032fda6af9bbc7628072175/B is initiating minor compaction (all files) 2024-11-21T01:45:08,404 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70b50c956032fda6af9bbc7628072175/B in TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:08,404 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/084f6609c933420db154e82695f3900e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/4f581e9ae9374d78b05b0a504c1790b1, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/c5f809db18ab4bf1882ae32b1b84c940] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp, totalSize=37.0 K 2024-11-21T01:45:08,404 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91cc41fcd7c040aabf00778d0190e50a, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=657, earliestPutTs=1732153504989 2024-11-21T01:45:08,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-21T01:45:08,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:08,408 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2902): Flushing 70b50c956032fda6af9bbc7628072175 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T01:45:08,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=A 2024-11-21T01:45:08,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:08,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=B 2024-11-21T01:45:08,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:08,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=C 2024-11-21T01:45:08,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:08,411 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 084f6609c933420db154e82695f3900e, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=657, earliestPutTs=1732153504989 2024-11-21T01:45:08,411 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab749c8e55b3438c961afa633e4dc854, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=676, earliestPutTs=1732153505341 2024-11-21T01:45:08,412 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f581e9ae9374d78b05b0a504c1790b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=676, earliestPutTs=1732153505341 2024-11-21T01:45:08,417 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ef3e606179f478685c59d09d1fe4740, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=692, earliestPutTs=1732153507505 2024-11-21T01:45:08,418 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting c5f809db18ab4bf1882ae32b1b84c940, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=692, earliestPutTs=1732153507505 2024-11-21T01:45:08,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/0f936fc3ab3a424e931dadf69b591157 is 50, key is test_row_0/A:col10/1732153507542/Put/seqid=0 2024-11-21T01:45:08,426 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70b50c956032fda6af9bbc7628072175#A#compaction#292 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:08,427 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/349c6afa7bef48f2aaf6c1ea6be0fb41 is 50, key is test_row_0/A:col10/1732153507505/Put/seqid=0 2024-11-21T01:45:08,428 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70b50c956032fda6af9bbc7628072175#B#compaction#293 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:08,428 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/e6fcfa4ee44a4f5a89f7c890b99a99e5 is 50, key is test_row_0/B:col10/1732153507505/Put/seqid=0 2024-11-21T01:45:08,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742160_1336 (size=12301) 2024-11-21T01:45:08,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742162_1338 (size=13391) 2024-11-21T01:45:08,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742161_1337 (size=13391) 2024-11-21T01:45:08,471 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/e6fcfa4ee44a4f5a89f7c890b99a99e5 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/e6fcfa4ee44a4f5a89f7c890b99a99e5 2024-11-21T01:45:08,474 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/349c6afa7bef48f2aaf6c1ea6be0fb41 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/349c6afa7bef48f2aaf6c1ea6be0fb41 2024-11-21T01:45:08,479 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70b50c956032fda6af9bbc7628072175/B of 70b50c956032fda6af9bbc7628072175 into e6fcfa4ee44a4f5a89f7c890b99a99e5(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:08,479 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:45:08,479 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., storeName=70b50c956032fda6af9bbc7628072175/B, priority=13, startTime=1732153508399; duration=0sec 2024-11-21T01:45:08,479 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:08,479 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:B 2024-11-21T01:45:08,479 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:08,482 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:08,482 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 70b50c956032fda6af9bbc7628072175/C is initiating minor compaction (all files) 2024-11-21T01:45:08,482 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70b50c956032fda6af9bbc7628072175/C in TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:08,482 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70b50c956032fda6af9bbc7628072175/A of 70b50c956032fda6af9bbc7628072175 into 349c6afa7bef48f2aaf6c1ea6be0fb41(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:08,482 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:45:08,482 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., storeName=70b50c956032fda6af9bbc7628072175/A, priority=13, startTime=1732153508399; duration=0sec 2024-11-21T01:45:08,482 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/aac2f6a8c5314dfc9241a911fc608f14, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/8effbce987dd4fc6ad426434ca6d2d71, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/91d04b169e364961b82d3b7a07331d74] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp, totalSize=37.0 K 2024-11-21T01:45:08,482 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:08,482 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:A 2024-11-21T01:45:08,483 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting aac2f6a8c5314dfc9241a911fc608f14, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=657, earliestPutTs=1732153504989 2024-11-21T01:45:08,485 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 8effbce987dd4fc6ad426434ca6d2d71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=676, earliestPutTs=1732153505341 2024-11-21T01:45:08,485 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 91d04b169e364961b82d3b7a07331d74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=692, earliestPutTs=1732153507505 2024-11-21T01:45:08,495 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70b50c956032fda6af9bbc7628072175#C#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:08,495 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/5dbc2b8a47e144c798e8b98fa06e2359 is 50, key is test_row_0/C:col10/1732153507505/Put/seqid=0 2024-11-21T01:45:08,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742163_1339 (size=13391) 2024-11-21T01:45:08,520 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/5dbc2b8a47e144c798e8b98fa06e2359 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/5dbc2b8a47e144c798e8b98fa06e2359 2024-11-21T01:45:08,548 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70b50c956032fda6af9bbc7628072175/C of 70b50c956032fda6af9bbc7628072175 into 5dbc2b8a47e144c798e8b98fa06e2359(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:08,548 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:45:08,548 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., storeName=70b50c956032fda6af9bbc7628072175/C, priority=13, startTime=1732153508399; duration=0sec 2024-11-21T01:45:08,548 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:08,548 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:C 2024-11-21T01:45:08,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-21T01:45:08,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 70b50c956032fda6af9bbc7628072175 2024-11-21T01:45:08,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:45:08,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:08,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153568707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:08,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:08,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153568711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:08,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:08,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:08,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:08,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:08,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153568712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:08,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:08,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153568716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:08,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:08,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:08,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:08,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:08,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:08,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:08,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:08,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153568823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:08,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:08,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153568823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:08,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:08,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:08,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:08,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:08,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153568823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:08,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:08,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:08,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:08,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:08,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153568823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:08,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:08,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:08,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:08,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:08,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:08,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:08,858 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=712 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/0f936fc3ab3a424e931dadf69b591157 2024-11-21T01:45:08,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/f619511b445c4c9d8834447b58c25fa4 is 50, key is test_row_0/B:col10/1732153507542/Put/seqid=0 2024-11-21T01:45:08,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742164_1340 (size=12301) 2024-11-21T01:45:08,898 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=712 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/f619511b445c4c9d8834447b58c25fa4 2024-11-21T01:45:08,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/d6ecf5bf1df943d9845f3039c77a360e is 50, key is test_row_0/C:col10/1732153507542/Put/seqid=0 2024-11-21T01:45:08,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742165_1341 (size=12301) 2024-11-21T01:45:09,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:09,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153569033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:09,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:09,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:09,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153569033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:09,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:09,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:09,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:09,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153569033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:09,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153569033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:09,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:09,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:09,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:09,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:09,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:09,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:09,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:09,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153569342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:09,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:09,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:09,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:09,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153569343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:09,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:09,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:09,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:09,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153569343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:09,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:09,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:09,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153569343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:09,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:09,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:09,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:09,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:09,362 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=712 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/d6ecf5bf1df943d9845f3039c77a360e 2024-11-21T01:45:09,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/0f936fc3ab3a424e931dadf69b591157 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/0f936fc3ab3a424e931dadf69b591157 2024-11-21T01:45:09,374 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/0f936fc3ab3a424e931dadf69b591157, entries=150, sequenceid=712, filesize=12.0 K 2024-11-21T01:45:09,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/f619511b445c4c9d8834447b58c25fa4 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/f619511b445c4c9d8834447b58c25fa4 2024-11-21T01:45:09,380 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/f619511b445c4c9d8834447b58c25fa4, entries=150, sequenceid=712, filesize=12.0 K 2024-11-21T01:45:09,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/d6ecf5bf1df943d9845f3039c77a360e as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/d6ecf5bf1df943d9845f3039c77a360e 2024-11-21T01:45:09,386 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/d6ecf5bf1df943d9845f3039c77a360e, entries=150, sequenceid=712, filesize=12.0 K 2024-11-21T01:45:09,387 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 70b50c956032fda6af9bbc7628072175 in 979ms, sequenceid=712, compaction requested=false 2024-11-21T01:45:09,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2603): Flush status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:45:09,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:09,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=89 2024-11-21T01:45:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=89 2024-11-21T01:45:09,391 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-21T01:45:09,391 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9110 sec 2024-11-21T01:45:09,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees in 1.9190 sec 2024-11-21T01:45:09,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-21T01:45:09,623 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:45:09,624 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:45:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees 2024-11-21T01:45:09,626 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:45:09,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-21T01:45:09,627 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:45:09,627 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:45:09,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-21T01:45:09,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-21T01:45:09,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:09,780 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2902): Flushing 70b50c956032fda6af9bbc7628072175 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T01:45:09,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=A 2024-11-21T01:45:09,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:09,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=B 2024-11-21T01:45:09,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:09,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=C 2024-11-21T01:45:09,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:09,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/3a27efbcb8684214a7f3eb2a5863e1e4 is 50, key is test_row_0/A:col10/1732153508708/Put/seqid=0 2024-11-21T01:45:09,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742166_1342 (size=12301) 2024-11-21T01:45:09,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 70b50c956032fda6af9bbc7628072175 2024-11-21T01:45:09,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:45:09,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:09,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153569873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:09,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:09,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:09,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:09,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:09,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153569875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:09,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:09,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:09,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:09,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:09,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153569877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:09,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:09,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153569878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:09,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:09,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:09,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:09,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-21T01:45:09,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:09,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153569983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:09,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:09,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153569983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:09,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:09,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:09,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:09,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:09,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:09,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:09,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:09,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153569994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:09,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:09,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:09,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:09,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153569994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:09,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:09,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:09,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:09,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:10,192 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=728 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/3a27efbcb8684214a7f3eb2a5863e1e4 2024-11-21T01:45:10,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:10,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153570193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:10,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:10,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:10,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:10,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:10,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153570193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:10,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:10,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:10,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:10,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/a87aed0d0536484bb59edd0dd9e40a3f is 50, key is test_row_0/B:col10/1732153508708/Put/seqid=0 2024-11-21T01:45:10,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742167_1343 (size=12301) 2024-11-21T01:45:10,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:10,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153570203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:10,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:10,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153570203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:10,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-21T01:45:10,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:10,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153570502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:10,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:10,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:10,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:10,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:10,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153570503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:10,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:10,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:10,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:10,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:10,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153570513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:10,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:10,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:10,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:10,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:10,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153570513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:10,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:10,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:10,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:10,605 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=728 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/a87aed0d0536484bb59edd0dd9e40a3f 2024-11-21T01:45:10,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/6b5debbc0bde4531b46ce2204e6f1261 is 50, key is test_row_0/C:col10/1732153508708/Put/seqid=0 2024-11-21T01:45:10,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742168_1344 (size=12301) 2024-11-21T01:45:10,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-21T01:45:11,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:11,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153571013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:11,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:11,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:11,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:11,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153571013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:11,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:11,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:11,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:11,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:11,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:11,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153571022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:11,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:11,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:11,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153571023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:11,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:11,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:11,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:11,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:11,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:11,078 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=728 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/6b5debbc0bde4531b46ce2204e6f1261 2024-11-21T01:45:11,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/3a27efbcb8684214a7f3eb2a5863e1e4 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/3a27efbcb8684214a7f3eb2a5863e1e4 2024-11-21T01:45:11,089 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/3a27efbcb8684214a7f3eb2a5863e1e4, entries=150, sequenceid=728, filesize=12.0 K 2024-11-21T01:45:11,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/a87aed0d0536484bb59edd0dd9e40a3f as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/a87aed0d0536484bb59edd0dd9e40a3f 2024-11-21T01:45:11,094 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/a87aed0d0536484bb59edd0dd9e40a3f, entries=150, sequenceid=728, filesize=12.0 K 2024-11-21T01:45:11,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/6b5debbc0bde4531b46ce2204e6f1261 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/6b5debbc0bde4531b46ce2204e6f1261 2024-11-21T01:45:11,099 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/6b5debbc0bde4531b46ce2204e6f1261, entries=150, sequenceid=728, filesize=12.0 K 2024-11-21T01:45:11,100 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 70b50c956032fda6af9bbc7628072175 in 1320ms, sequenceid=728, compaction requested=true 2024-11-21T01:45:11,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2603): Flush status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:45:11,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:11,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=91 2024-11-21T01:45:11,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=91 2024-11-21T01:45:11,104 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-21T01:45:11,104 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4750 sec 2024-11-21T01:45:11,106 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees in 1.4810 sec 2024-11-21T01:45:11,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-21T01:45:11,773 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:45:11,774 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:45:11,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees 2024-11-21T01:45:11,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-21T01:45:11,776 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=92, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:45:11,777 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=92, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:45:11,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:45:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-21T01:45:11,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-11-21T01:45:11,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:11,930 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2902): Flushing 70b50c956032fda6af9bbc7628072175 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T01:45:11,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=A 2024-11-21T01:45:11,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:11,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=B 2024-11-21T01:45:11,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:11,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=C 2024-11-21T01:45:11,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:11,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/5fc0091a15574c9eacc04770de9b4a14 is 50, key is test_row_0/A:col10/1732153509869/Put/seqid=0 2024-11-21T01:45:11,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742169_1345 (size=12301) 2024-11-21T01:45:12,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 70b50c956032fda6af9bbc7628072175 2024-11-21T01:45:12,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. as already flushing 2024-11-21T01:45:12,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153572073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:12,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:12,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153572073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:12,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153572074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:12,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153572080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:12,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153572080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-21T01:45:12,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153572183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153572183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153572183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:12,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:12,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153572192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:12,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153572192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:12,201 INFO [Thread-1130 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1130. 2024-11-21T01:45:12,201 DEBUG [Thread-1130 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:45:12,201 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:12,201 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:12,201 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:45:12,203 INFO [Thread-1126 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1126. 2024-11-21T01:45:12,203 DEBUG [Thread-1126 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:45:12,203 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:12,204 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:45:12,204 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:12,205 INFO [Thread-1134 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1134. 2024-11-21T01:45:12,205 DEBUG [Thread-1134 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:45:12,205 DEBUG [Thread-1134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:12,205 DEBUG [Thread-1134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:12,205 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:45:12,206 INFO [Thread-1128 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1128. 2024-11-21T01:45:12,206 DEBUG [Thread-1128 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:45:12,206 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:12,206 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:12,206 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:45:12,208 INFO [Thread-1132 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1132. 2024-11-21T01:45:12,209 DEBUG [Thread-1132 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:45:12,209 DEBUG [Thread-1132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:12,209 DEBUG [Thread-1132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:12,209 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:45:12,343 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=747 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/5fc0091a15574c9eacc04770de9b4a14 2024-11-21T01:45:12,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/b68bdc81bff74c4c992b10d532cfd2f3 is 50, key is test_row_0/B:col10/1732153509869/Put/seqid=0 2024-11-21T01:45:12,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742170_1346 (size=12301) 2024-11-21T01:45:12,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:12,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:12,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:12,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153572393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:12,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153572393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:12,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153572393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:12,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:12,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:12,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:12,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-21T01:45:12,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:12,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153572402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153572402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:12,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:12,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:12,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:12,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:12,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:12,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54092 deadline: 1732153572703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:12,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54090 deadline: 1732153572703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:12,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54058 deadline: 1732153572703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:12,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:12,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:12,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:12,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:12,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:12,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54056 deadline: 1732153572713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:12,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54046 deadline: 1732153572713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:12,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:12,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:12,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:12,753 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=747 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/b68bdc81bff74c4c992b10d532cfd2f3 2024-11-21T01:45:12,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/3ce4c2a90c284693a0ff3b51ab6c00cb is 50, key is test_row_0/C:col10/1732153509869/Put/seqid=0 2024-11-21T01:45:12,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742171_1347 (size=12301) 2024-11-21T01:45:12,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-21T01:45:13,165 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=747 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/3ce4c2a90c284693a0ff3b51ab6c00cb 2024-11-21T01:45:13,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/5fc0091a15574c9eacc04770de9b4a14 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/5fc0091a15574c9eacc04770de9b4a14 2024-11-21T01:45:13,171 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/5fc0091a15574c9eacc04770de9b4a14, entries=150, sequenceid=747, filesize=12.0 K 2024-11-21T01:45:13,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/b68bdc81bff74c4c992b10d532cfd2f3 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/b68bdc81bff74c4c992b10d532cfd2f3 2024-11-21T01:45:13,175 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/b68bdc81bff74c4c992b10d532cfd2f3, entries=150, sequenceid=747, filesize=12.0 K 2024-11-21T01:45:13,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/3ce4c2a90c284693a0ff3b51ab6c00cb as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/3ce4c2a90c284693a0ff3b51ab6c00cb 2024-11-21T01:45:13,178 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/3ce4c2a90c284693a0ff3b51ab6c00cb, entries=150, sequenceid=747, filesize=12.0 K 2024-11-21T01:45:13,179 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 70b50c956032fda6af9bbc7628072175 in 1248ms, sequenceid=747, compaction requested=true 2024-11-21T01:45:13,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2603): Flush status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:45:13,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:13,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=93 2024-11-21T01:45:13,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=93 2024-11-21T01:45:13,181 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-21T01:45:13,181 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4030 sec 2024-11-21T01:45:13,183 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees in 1.4080 sec 2024-11-21T01:45:13,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 70b50c956032fda6af9bbc7628072175 2024-11-21T01:45:13,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70b50c956032fda6af9bbc7628072175 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-21T01:45:13,215 INFO [Thread-1119 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1119. 2024-11-21T01:45:13,215 INFO [Thread-1121 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1121. 2024-11-21T01:45:13,215 INFO [Thread-1123 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1123. 2024-11-21T01:45:13,215 DEBUG [Thread-1119 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:45:13,215 DEBUG [Thread-1123 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:45:13,215 DEBUG [Thread-1121 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:45:13,215 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:13,215 DEBUG [Thread-1123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:13,215 DEBUG [Thread-1121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:13,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=A 2024-11-21T01:45:13,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:13,215 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:13,215 DEBUG [Thread-1121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:13,215 DEBUG [Thread-1123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:13,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=B 2024-11-21T01:45:13,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:13,215 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:45:13,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=C 2024-11-21T01:45:13,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:13,215 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:45:13,215 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:45:13,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/72ed3722ce254cd399ea7b02264af5d2 is 50, key is test_row_0/A:col10/1732153512071/Put/seqid=0 2024-11-21T01:45:13,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742172_1348 (size=12301) 2024-11-21T01:45:13,224 INFO [Thread-1115 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1115. 2024-11-21T01:45:13,224 DEBUG [Thread-1115 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:45:13,224 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:13,224 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:13,224 INFO [Thread-1117 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1117. 2024-11-21T01:45:13,224 DEBUG [Thread-1117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:45:13,224 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:45:13,225 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:13,225 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:13,225 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:45:13,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=763 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/72ed3722ce254cd399ea7b02264af5d2 2024-11-21T01:45:13,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/12c0f63059784f3f8f3aa35584d91b35 is 50, key is test_row_0/B:col10/1732153512071/Put/seqid=0 2024-11-21T01:45:13,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742173_1349 (size=12301) 2024-11-21T01:45:13,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-21T01:45:13,923 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:45:13,923 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T01:45:13,923 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 140 2024-11-21T01:45:13,923 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 143 2024-11-21T01:45:13,923 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 137 2024-11-21T01:45:13,923 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 149 2024-11-21T01:45:13,923 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-21T01:45:13,923 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T01:45:13,923 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2635 2024-11-21T01:45:13,923 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2617 2024-11-21T01:45:13,923 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2556 2024-11-21T01:45:13,924 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2606 2024-11-21T01:45:13,924 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2613 2024-11-21T01:45:13,924 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T01:45:13,924 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T01:45:13,924 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T01:45:13,924 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:13,924 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:13,924 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:45:13,925 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-21T01:45:13,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T01:45:13,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-21T01:45:13,928 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732153513928"}]},"ts":"1732153513928"} 2024-11-21T01:45:13,930 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T01:45:13,930 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T01:45:13,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T01:45:13,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=70b50c956032fda6af9bbc7628072175, UNASSIGN}] 2024-11-21T01:45:13,933 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=70b50c956032fda6af9bbc7628072175, UNASSIGN 2024-11-21T01:45:13,934 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=70b50c956032fda6af9bbc7628072175, regionState=CLOSING, regionLocation=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:13,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=70b50c956032fda6af9bbc7628072175, UNASSIGN because future has completed 2024-11-21T01:45:13,935 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T01:45:13,935 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359}] 2024-11-21T01:45:14,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=763 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/12c0f63059784f3f8f3aa35584d91b35 2024-11-21T01:45:14,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-21T01:45:14,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/51181d0663984f36a2e460dee7dbd8f8 is 50, key is test_row_0/C:col10/1732153512071/Put/seqid=0 2024-11-21T01:45:14,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742174_1350 (size=12301) 2024-11-21T01:45:14,087 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(122): Close 70b50c956032fda6af9bbc7628072175 2024-11-21T01:45:14,088 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T01:45:14,088 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1722): Closing 70b50c956032fda6af9bbc7628072175, disabling compactions & flushes 2024-11-21T01:45:14,088 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:14,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-21T01:45:14,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=763 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/51181d0663984f36a2e460dee7dbd8f8 2024-11-21T01:45:14,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/72ed3722ce254cd399ea7b02264af5d2 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/72ed3722ce254cd399ea7b02264af5d2 2024-11-21T01:45:14,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/72ed3722ce254cd399ea7b02264af5d2, entries=150, sequenceid=763, filesize=12.0 K 2024-11-21T01:45:14,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/12c0f63059784f3f8f3aa35584d91b35 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/12c0f63059784f3f8f3aa35584d91b35 2024-11-21T01:45:14,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/12c0f63059784f3f8f3aa35584d91b35, entries=150, sequenceid=763, filesize=12.0 K 2024-11-21T01:45:14,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/51181d0663984f36a2e460dee7dbd8f8 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/51181d0663984f36a2e460dee7dbd8f8 2024-11-21T01:45:14,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/51181d0663984f36a2e460dee7dbd8f8, entries=150, sequenceid=763, filesize=12.0 K 2024-11-21T01:45:14,458 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=13.42 KB/13740 for 70b50c956032fda6af9bbc7628072175 in 1244ms, sequenceid=763, compaction requested=true 2024-11-21T01:45:14,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70b50c956032fda6af9bbc7628072175: 2024-11-21T01:45:14,458 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:14,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:45:14,458 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:14,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:14,458 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. because compaction request was cancelled 2024-11-21T01:45:14,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:45:14,458 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:A 2024-11-21T01:45:14,458 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. after waiting 0 ms 2024-11-21T01:45:14,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:14,458 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:14,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70b50c956032fda6af9bbc7628072175:C, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:45:14,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:14,458 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. because compaction request was cancelled 2024-11-21T01:45:14,458 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. because compaction request was cancelled 2024-11-21T01:45:14,458 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:C 2024-11-21T01:45:14,458 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70b50c956032fda6af9bbc7628072175:B 2024-11-21T01:45:14,458 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(2902): Flushing 70b50c956032fda6af9bbc7628072175 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-21T01:45:14,459 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=A 2024-11-21T01:45:14,459 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:14,459 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=B 2024-11-21T01:45:14,459 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:14,459 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70b50c956032fda6af9bbc7628072175, store=C 2024-11-21T01:45:14,459 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:14,462 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/d4877e7f666446bdb6f333c0ec145c01 is 50, key is test_row_2/A:col10/1732153513223/Put/seqid=0 2024-11-21T01:45:14,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742175_1351 (size=7415) 2024-11-21T01:45:14,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-21T01:45:14,866 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=768 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/d4877e7f666446bdb6f333c0ec145c01 2024-11-21T01:45:14,872 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/2bfa2856bed6427b8880e320d9664e4d is 50, key is test_row_2/B:col10/1732153513223/Put/seqid=0 2024-11-21T01:45:14,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742176_1352 (size=7415) 2024-11-21T01:45:15,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-21T01:45:15,276 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=768 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/2bfa2856bed6427b8880e320d9664e4d 2024-11-21T01:45:15,282 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/af93d7703bee4f818eab670896742f39 is 50, key is test_row_2/C:col10/1732153513223/Put/seqid=0 2024-11-21T01:45:15,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742177_1353 (size=7415) 2024-11-21T01:45:15,686 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=768 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/af93d7703bee4f818eab670896742f39 2024-11-21T01:45:15,690 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/A/d4877e7f666446bdb6f333c0ec145c01 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/d4877e7f666446bdb6f333c0ec145c01 2024-11-21T01:45:15,693 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/d4877e7f666446bdb6f333c0ec145c01, entries=50, sequenceid=768, filesize=7.2 K 2024-11-21T01:45:15,694 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/B/2bfa2856bed6427b8880e320d9664e4d as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/2bfa2856bed6427b8880e320d9664e4d 2024-11-21T01:45:15,697 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/2bfa2856bed6427b8880e320d9664e4d, entries=50, sequenceid=768, filesize=7.2 K 2024-11-21T01:45:15,697 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/.tmp/C/af93d7703bee4f818eab670896742f39 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/af93d7703bee4f818eab670896742f39 2024-11-21T01:45:15,700 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/af93d7703bee4f818eab670896742f39, entries=50, sequenceid=768, filesize=7.2 K 2024-11-21T01:45:15,701 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 70b50c956032fda6af9bbc7628072175 in 1243ms, sequenceid=768, compaction requested=true 2024-11-21T01:45:15,701 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/2057b9bedf78445f8c3f9832dd8b5d89, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/0382280c8f654cf899c6fa3f9bc5a54e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/bff4413918d043899da651dd861282b3, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/2a4d722b47b045e7b2b3eee325d9ab3f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/dd5bd9b0d05e41c3be957c468cb82bc2, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/29789c06a5854fc3984761431766df3f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/16bf1b1610fb431e82ed24288be51fa3, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/b4f95049b08940ccab76bd0953671290, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/138c318c37b24367bf98ef882ae3072a, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/b4741c5b1ec84516ab3091bb7e1c890c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/0ba32bf22f494af8a278df47e993619a, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/ad16f5364b0c474bba146ceb06a5eba4, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/027693d37f4f41f199cedf2b91239a3a, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/3dac2a72b1d74c0aa24d1f881fde73de, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/37e87f4aa0b8459c87983f92a7935341, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/e0dbfbe3180d484197bcfd7b8d19a05c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/e5665933e09a4d859da80c5343ca066c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/aaa6db9d19874414a3dcd8001d264eee, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/0d2e89cff2c1484fb113de30029ede9e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/46729d28357c42ba823d6d8356132dd5, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/89a9b2e50c0543be85c651a47a198e8f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/e6462b64ca9044a9bbce404baf5de3a5, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/1ee94be136c1452abc6d696e7604aef5, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/8d353fa1104442feb043c23b711ca447, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/c27a2f3be6864544bf18fb2e0d0c477f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/0158e6c8dcdb47159d04b0f7ea2a8187, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/c147c1af44184fdf8fd84481efd47a6e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/b58f0673fb3546839dd12b1309b59217, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/91cc41fcd7c040aabf00778d0190e50a, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/b778c726a9d34d95a37cce35928618ad, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/ab749c8e55b3438c961afa633e4dc854, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/0ef3e606179f478685c59d09d1fe4740] to archive 2024-11-21T01:45:15,702 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T01:45:15,703 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/2057b9bedf78445f8c3f9832dd8b5d89 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/2057b9bedf78445f8c3f9832dd8b5d89 2024-11-21T01:45:15,704 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/0382280c8f654cf899c6fa3f9bc5a54e to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/0382280c8f654cf899c6fa3f9bc5a54e 2024-11-21T01:45:15,705 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/bff4413918d043899da651dd861282b3 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/bff4413918d043899da651dd861282b3 2024-11-21T01:45:15,706 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/2a4d722b47b045e7b2b3eee325d9ab3f to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/2a4d722b47b045e7b2b3eee325d9ab3f 2024-11-21T01:45:15,707 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/dd5bd9b0d05e41c3be957c468cb82bc2 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/dd5bd9b0d05e41c3be957c468cb82bc2 2024-11-21T01:45:15,707 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/29789c06a5854fc3984761431766df3f to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/29789c06a5854fc3984761431766df3f 2024-11-21T01:45:15,708 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/16bf1b1610fb431e82ed24288be51fa3 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/16bf1b1610fb431e82ed24288be51fa3 2024-11-21T01:45:15,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/b4f95049b08940ccab76bd0953671290 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/b4f95049b08940ccab76bd0953671290 2024-11-21T01:45:15,710 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/138c318c37b24367bf98ef882ae3072a to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/138c318c37b24367bf98ef882ae3072a 2024-11-21T01:45:15,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/b4741c5b1ec84516ab3091bb7e1c890c to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/b4741c5b1ec84516ab3091bb7e1c890c 2024-11-21T01:45:15,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/0ba32bf22f494af8a278df47e993619a to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/0ba32bf22f494af8a278df47e993619a 2024-11-21T01:45:15,712 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/ad16f5364b0c474bba146ceb06a5eba4 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/ad16f5364b0c474bba146ceb06a5eba4 2024-11-21T01:45:15,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/027693d37f4f41f199cedf2b91239a3a to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/027693d37f4f41f199cedf2b91239a3a 2024-11-21T01:45:15,714 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/3dac2a72b1d74c0aa24d1f881fde73de to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/3dac2a72b1d74c0aa24d1f881fde73de 2024-11-21T01:45:15,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/37e87f4aa0b8459c87983f92a7935341 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/37e87f4aa0b8459c87983f92a7935341 2024-11-21T01:45:15,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/e0dbfbe3180d484197bcfd7b8d19a05c to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/e0dbfbe3180d484197bcfd7b8d19a05c 2024-11-21T01:45:15,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/e5665933e09a4d859da80c5343ca066c to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/e5665933e09a4d859da80c5343ca066c 2024-11-21T01:45:15,717 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/aaa6db9d19874414a3dcd8001d264eee to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/aaa6db9d19874414a3dcd8001d264eee 2024-11-21T01:45:15,718 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/0d2e89cff2c1484fb113de30029ede9e to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/0d2e89cff2c1484fb113de30029ede9e 2024-11-21T01:45:15,719 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/46729d28357c42ba823d6d8356132dd5 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/46729d28357c42ba823d6d8356132dd5 2024-11-21T01:45:15,719 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/89a9b2e50c0543be85c651a47a198e8f to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/89a9b2e50c0543be85c651a47a198e8f 2024-11-21T01:45:15,720 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/e6462b64ca9044a9bbce404baf5de3a5 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/e6462b64ca9044a9bbce404baf5de3a5 2024-11-21T01:45:15,721 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/1ee94be136c1452abc6d696e7604aef5 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/1ee94be136c1452abc6d696e7604aef5 2024-11-21T01:45:15,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/8d353fa1104442feb043c23b711ca447 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/8d353fa1104442feb043c23b711ca447 2024-11-21T01:45:15,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/c27a2f3be6864544bf18fb2e0d0c477f to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/c27a2f3be6864544bf18fb2e0d0c477f 2024-11-21T01:45:15,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/0158e6c8dcdb47159d04b0f7ea2a8187 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/0158e6c8dcdb47159d04b0f7ea2a8187 2024-11-21T01:45:15,724 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/c147c1af44184fdf8fd84481efd47a6e to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/c147c1af44184fdf8fd84481efd47a6e 2024-11-21T01:45:15,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/b58f0673fb3546839dd12b1309b59217 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/b58f0673fb3546839dd12b1309b59217 2024-11-21T01:45:15,726 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/91cc41fcd7c040aabf00778d0190e50a to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/91cc41fcd7c040aabf00778d0190e50a 2024-11-21T01:45:15,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/b778c726a9d34d95a37cce35928618ad to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/b778c726a9d34d95a37cce35928618ad 2024-11-21T01:45:15,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/ab749c8e55b3438c961afa633e4dc854 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/ab749c8e55b3438c961afa633e4dc854 2024-11-21T01:45:15,728 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/0ef3e606179f478685c59d09d1fe4740 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/0ef3e606179f478685c59d09d1fe4740 2024-11-21T01:45:15,730 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/6386a7cdd69340b8ac333bb0575c9f5f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/84c803a71ec7468f9ef44b65ae462a7d, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/19695532f5d14170901ab4aebeb265f4, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/05af23124ea4486ab53ede1011059e65, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/7c133d3e231d4ff890ece2db513c17e9, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/9c4f04de156a43eaa506cfb5d49084be, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/35b62a1a6f774a2a9988dbe205bc46df, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/2d988ab51a694698a71c88330ae7de7b, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/57bb121f562f40f9b90a7e8f9315792d, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/bd59044c5fb248eab6aaaef4e76d6d35, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/f17cf062a61a4a7484ea4db6681ecb60, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/cf606f2b91de41648874c64b31c17eb6, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/007c916ae47f4694b2863afddc13f290, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/a89663ac813c415bab1e88db50a4bcd9, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/850b1f47bc9c4ba0b2cbba78098f4807, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/4295f090220140c29d189290084671ae, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/0ca139f168504054836ad36110413a93, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/717b5232cc9b476b8710063459aca70e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/821212b4c0a6427c909e71e95d8b2fb0, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/4fb4f22e9f2644378c2e4d7d273e5577, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/eb7dfd4899234b6d9747db6745380deb, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/afd258ce1875486c92e5716e42fccff5, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/fa1470e836f24a8aa6b551ff94b2a2eb, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/33ac50410a36439888b276f17f07c42c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/aeb501fc178846aa831ab29bae111ca9, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/505e70b1c7864ebcab84619e706c22c3, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/e1f0f02eb473460f9b10c6e73d77f336, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/a1e7493a0cf145a59e9ffc344a751d4f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/084f6609c933420db154e82695f3900e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/27b46b4d33b24243be8ad7f32b0d0b0e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/4f581e9ae9374d78b05b0a504c1790b1, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/c5f809db18ab4bf1882ae32b1b84c940] to archive 2024-11-21T01:45:15,730 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T01:45:15,731 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/6386a7cdd69340b8ac333bb0575c9f5f to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/6386a7cdd69340b8ac333bb0575c9f5f 2024-11-21T01:45:15,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/84c803a71ec7468f9ef44b65ae462a7d to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/84c803a71ec7468f9ef44b65ae462a7d 2024-11-21T01:45:15,733 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/19695532f5d14170901ab4aebeb265f4 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/19695532f5d14170901ab4aebeb265f4 2024-11-21T01:45:15,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/05af23124ea4486ab53ede1011059e65 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/05af23124ea4486ab53ede1011059e65 2024-11-21T01:45:15,735 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/7c133d3e231d4ff890ece2db513c17e9 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/7c133d3e231d4ff890ece2db513c17e9 2024-11-21T01:45:15,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/9c4f04de156a43eaa506cfb5d49084be to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/9c4f04de156a43eaa506cfb5d49084be 2024-11-21T01:45:15,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/35b62a1a6f774a2a9988dbe205bc46df to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/35b62a1a6f774a2a9988dbe205bc46df 2024-11-21T01:45:15,738 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/2d988ab51a694698a71c88330ae7de7b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/2d988ab51a694698a71c88330ae7de7b 2024-11-21T01:45:15,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/57bb121f562f40f9b90a7e8f9315792d to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/57bb121f562f40f9b90a7e8f9315792d 2024-11-21T01:45:15,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/bd59044c5fb248eab6aaaef4e76d6d35 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/bd59044c5fb248eab6aaaef4e76d6d35 2024-11-21T01:45:15,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/f17cf062a61a4a7484ea4db6681ecb60 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/f17cf062a61a4a7484ea4db6681ecb60 2024-11-21T01:45:15,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/cf606f2b91de41648874c64b31c17eb6 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/cf606f2b91de41648874c64b31c17eb6 2024-11-21T01:45:15,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/007c916ae47f4694b2863afddc13f290 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/007c916ae47f4694b2863afddc13f290 2024-11-21T01:45:15,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/a89663ac813c415bab1e88db50a4bcd9 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/a89663ac813c415bab1e88db50a4bcd9 2024-11-21T01:45:15,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/850b1f47bc9c4ba0b2cbba78098f4807 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/850b1f47bc9c4ba0b2cbba78098f4807 2024-11-21T01:45:15,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/4295f090220140c29d189290084671ae to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/4295f090220140c29d189290084671ae 2024-11-21T01:45:15,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/0ca139f168504054836ad36110413a93 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/0ca139f168504054836ad36110413a93 2024-11-21T01:45:15,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/717b5232cc9b476b8710063459aca70e to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/717b5232cc9b476b8710063459aca70e 2024-11-21T01:45:15,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/821212b4c0a6427c909e71e95d8b2fb0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/821212b4c0a6427c909e71e95d8b2fb0 2024-11-21T01:45:15,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/4fb4f22e9f2644378c2e4d7d273e5577 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/4fb4f22e9f2644378c2e4d7d273e5577 2024-11-21T01:45:15,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/eb7dfd4899234b6d9747db6745380deb to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/eb7dfd4899234b6d9747db6745380deb 2024-11-21T01:45:15,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/afd258ce1875486c92e5716e42fccff5 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/afd258ce1875486c92e5716e42fccff5 2024-11-21T01:45:15,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/fa1470e836f24a8aa6b551ff94b2a2eb to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/fa1470e836f24a8aa6b551ff94b2a2eb 2024-11-21T01:45:15,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/33ac50410a36439888b276f17f07c42c to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/33ac50410a36439888b276f17f07c42c 2024-11-21T01:45:15,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/aeb501fc178846aa831ab29bae111ca9 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/aeb501fc178846aa831ab29bae111ca9 2024-11-21T01:45:15,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/505e70b1c7864ebcab84619e706c22c3 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/505e70b1c7864ebcab84619e706c22c3 2024-11-21T01:45:15,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/e1f0f02eb473460f9b10c6e73d77f336 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/e1f0f02eb473460f9b10c6e73d77f336 2024-11-21T01:45:15,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/a1e7493a0cf145a59e9ffc344a751d4f to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/a1e7493a0cf145a59e9ffc344a751d4f 2024-11-21T01:45:15,756 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/084f6609c933420db154e82695f3900e to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/084f6609c933420db154e82695f3900e 2024-11-21T01:45:15,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/27b46b4d33b24243be8ad7f32b0d0b0e to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/27b46b4d33b24243be8ad7f32b0d0b0e 2024-11-21T01:45:15,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/4f581e9ae9374d78b05b0a504c1790b1 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/4f581e9ae9374d78b05b0a504c1790b1 2024-11-21T01:45:15,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/c5f809db18ab4bf1882ae32b1b84c940 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/c5f809db18ab4bf1882ae32b1b84c940 2024-11-21T01:45:15,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/9217a03ebff34efc8eb0c1105649a884, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/115201f373824d6abd3f91a320249a73, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/c6d94948cbb148a398799781f3bae406, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/cd8c40677dc44cb099f0b2a7ccf4326c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/150a426bb5fe4efebc2399a223021896, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/2077a936b78d42638c94f15adfb22658, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/b778b520e41d48c6ba15268d16577a20, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/f2d305ecdc424b8bb6f05cdc84bc5a7c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/ded6e9fdd7dc4b0e9237a09ea4fe31e9, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/e6cfeca440e24507ac225f09036fa521, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/d5827c07617e44888ba305024b20cc38, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/b0c3ac6323b74fad9f0bf298f94d1cee, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/1baf8419e9e64f0d928095d32b3d0acd, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/93bb8b71c2cc4c2c80612e5d0db7e636, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/6ccde64db3764c9097d67359fdd3ba35, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/2a6f9ae41de64a36bd2b63549a0ddd85, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/7b23ea0149d249069483e03639a973e3, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/c4b010a4b9e643b1b0d7806538caa0f1, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/d58844f988ab435ba06ee7facf203b27, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/611c615ec5004ca48dc29bebf1dd428e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/74eb05e4becf444b91332cde9a1b5bdb, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/ddc05aa2fb4c490ba2848306131304cd, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/a417be84c3e348259a62baece25260ef, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/1a7ce430785d492b8b7164178ec17148, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/cbacd76999af4f5abb916221139c4a40, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/ea8b86b0e37c4f09a08c41b80f968035, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/5680889a2a9e4ca5828b800aa7b2860a, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/c6f53093e29e41c0b8973c7564d460fb, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/aac2f6a8c5314dfc9241a911fc608f14, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/e432aabf0f5344e6b91406e1ded2c765, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/8effbce987dd4fc6ad426434ca6d2d71, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/91d04b169e364961b82d3b7a07331d74] to archive 2024-11-21T01:45:15,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T01:45:15,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/9217a03ebff34efc8eb0c1105649a884 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/9217a03ebff34efc8eb0c1105649a884 2024-11-21T01:45:15,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/115201f373824d6abd3f91a320249a73 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/115201f373824d6abd3f91a320249a73 2024-11-21T01:45:15,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/c6d94948cbb148a398799781f3bae406 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/c6d94948cbb148a398799781f3bae406 2024-11-21T01:45:15,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/cd8c40677dc44cb099f0b2a7ccf4326c to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/cd8c40677dc44cb099f0b2a7ccf4326c 2024-11-21T01:45:15,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/150a426bb5fe4efebc2399a223021896 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/150a426bb5fe4efebc2399a223021896 2024-11-21T01:45:15,766 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/2077a936b78d42638c94f15adfb22658 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/2077a936b78d42638c94f15adfb22658 2024-11-21T01:45:15,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/b778b520e41d48c6ba15268d16577a20 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/b778b520e41d48c6ba15268d16577a20 2024-11-21T01:45:15,768 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/f2d305ecdc424b8bb6f05cdc84bc5a7c to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/f2d305ecdc424b8bb6f05cdc84bc5a7c 2024-11-21T01:45:15,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/ded6e9fdd7dc4b0e9237a09ea4fe31e9 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/ded6e9fdd7dc4b0e9237a09ea4fe31e9 2024-11-21T01:45:15,770 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/e6cfeca440e24507ac225f09036fa521 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/e6cfeca440e24507ac225f09036fa521 2024-11-21T01:45:15,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/d5827c07617e44888ba305024b20cc38 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/d5827c07617e44888ba305024b20cc38 2024-11-21T01:45:15,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/b0c3ac6323b74fad9f0bf298f94d1cee to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/b0c3ac6323b74fad9f0bf298f94d1cee 2024-11-21T01:45:15,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/1baf8419e9e64f0d928095d32b3d0acd to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/1baf8419e9e64f0d928095d32b3d0acd 2024-11-21T01:45:15,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/93bb8b71c2cc4c2c80612e5d0db7e636 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/93bb8b71c2cc4c2c80612e5d0db7e636 2024-11-21T01:45:15,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/6ccde64db3764c9097d67359fdd3ba35 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/6ccde64db3764c9097d67359fdd3ba35 2024-11-21T01:45:15,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/2a6f9ae41de64a36bd2b63549a0ddd85 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/2a6f9ae41de64a36bd2b63549a0ddd85 2024-11-21T01:45:15,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/7b23ea0149d249069483e03639a973e3 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/7b23ea0149d249069483e03639a973e3 2024-11-21T01:45:15,778 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/c4b010a4b9e643b1b0d7806538caa0f1 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/c4b010a4b9e643b1b0d7806538caa0f1 2024-11-21T01:45:15,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/d58844f988ab435ba06ee7facf203b27 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/d58844f988ab435ba06ee7facf203b27 2024-11-21T01:45:15,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/611c615ec5004ca48dc29bebf1dd428e to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/611c615ec5004ca48dc29bebf1dd428e 2024-11-21T01:45:15,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/74eb05e4becf444b91332cde9a1b5bdb to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/74eb05e4becf444b91332cde9a1b5bdb 2024-11-21T01:45:15,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/ddc05aa2fb4c490ba2848306131304cd to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/ddc05aa2fb4c490ba2848306131304cd 2024-11-21T01:45:15,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/a417be84c3e348259a62baece25260ef to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/a417be84c3e348259a62baece25260ef 2024-11-21T01:45:15,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/1a7ce430785d492b8b7164178ec17148 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/1a7ce430785d492b8b7164178ec17148 2024-11-21T01:45:15,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/cbacd76999af4f5abb916221139c4a40 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/cbacd76999af4f5abb916221139c4a40 2024-11-21T01:45:15,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/ea8b86b0e37c4f09a08c41b80f968035 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/ea8b86b0e37c4f09a08c41b80f968035 2024-11-21T01:45:15,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/5680889a2a9e4ca5828b800aa7b2860a to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/5680889a2a9e4ca5828b800aa7b2860a 2024-11-21T01:45:15,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/c6f53093e29e41c0b8973c7564d460fb to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/c6f53093e29e41c0b8973c7564d460fb 2024-11-21T01:45:15,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/aac2f6a8c5314dfc9241a911fc608f14 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/aac2f6a8c5314dfc9241a911fc608f14 2024-11-21T01:45:15,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/e432aabf0f5344e6b91406e1ded2c765 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/e432aabf0f5344e6b91406e1ded2c765 2024-11-21T01:45:15,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/8effbce987dd4fc6ad426434ca6d2d71 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/8effbce987dd4fc6ad426434ca6d2d71 2024-11-21T01:45:15,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/91d04b169e364961b82d3b7a07331d74 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/91d04b169e364961b82d3b7a07331d74 2024-11-21T01:45:15,793 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/recovered.edits/771.seqid, newMaxSeqId=771, maxSeqId=1 2024-11-21T01:45:15,794 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. 2024-11-21T01:45:15,794 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1676): Region close journal for 70b50c956032fda6af9bbc7628072175: Waiting for close lock at 1732153514088Running coprocessor pre-close hooks at 1732153514088Disabling compacts and flushes for region at 1732153514088Disabling writes for close at 1732153514458 (+370 ms)Obtaining lock to block concurrent updates at 1732153514458Preparing flush snapshotting stores in 70b50c956032fda6af9bbc7628072175 at 1732153514458Finished memstore snapshotting TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1732153514459 (+1 ms)Flushing stores of TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175. at 1732153514459Flushing 70b50c956032fda6af9bbc7628072175/A: creating writer at 1732153514459Flushing 70b50c956032fda6af9bbc7628072175/A: appending metadata at 1732153514461 (+2 ms)Flushing 70b50c956032fda6af9bbc7628072175/A: closing flushed file at 1732153514461Flushing 70b50c956032fda6af9bbc7628072175/B: creating writer at 1732153514870 (+409 ms)Flushing 70b50c956032fda6af9bbc7628072175/B: appending metadata at 1732153514872 (+2 ms)Flushing 70b50c956032fda6af9bbc7628072175/B: closing flushed file at 1732153514872Flushing 70b50c956032fda6af9bbc7628072175/C: creating writer at 1732153515279 (+407 ms)Flushing 70b50c956032fda6af9bbc7628072175/C: appending metadata at 1732153515281 (+2 ms)Flushing 70b50c956032fda6af9bbc7628072175/C: closing flushed file at 1732153515281Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e6d23e0: reopening flushed file at 1732153515689 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26ae92de: reopening flushed file at 1732153515693 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@118c6df7: reopening flushed file at 1732153515697 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 70b50c956032fda6af9bbc7628072175 in 1243ms, sequenceid=768, compaction requested=true at 1732153515701 (+4 ms)Writing region close event to WAL at 1732153515791 (+90 ms)Running coprocessor post-close hooks at 1732153515794 (+3 ms)Closed at 1732153515794 2024-11-21T01:45:15,795 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(157): Closed 70b50c956032fda6af9bbc7628072175 2024-11-21T01:45:15,796 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=70b50c956032fda6af9bbc7628072175, regionState=CLOSED 2024-11-21T01:45:15,797 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 because future has completed 2024-11-21T01:45:15,799 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-21T01:45:15,799 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseRegionProcedure 70b50c956032fda6af9bbc7628072175, server=b93ada835ef8,41129,1732153434359 in 1.8630 sec 2024-11-21T01:45:15,800 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-21T01:45:15,800 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=70b50c956032fda6af9bbc7628072175, UNASSIGN in 1.8670 sec 2024-11-21T01:45:15,802 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-21T01:45:15,802 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8710 sec 2024-11-21T01:45:15,803 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732153515803"}]},"ts":"1732153515803"} 2024-11-21T01:45:15,804 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T01:45:15,804 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T01:45:15,805 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8790 sec 2024-11-21T01:45:16,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-21T01:45:16,073 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:45:16,074 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-21T01:45:16,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=98, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:45:16,075 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=98, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:45:16,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-11-21T01:45:16,076 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=98, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:45:16,079 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175 2024-11-21T01:45:16,081 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A, FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B, FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C, FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/recovered.edits] 2024-11-21T01:45:16,083 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/0f936fc3ab3a424e931dadf69b591157 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/0f936fc3ab3a424e931dadf69b591157 2024-11-21T01:45:16,085 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/349c6afa7bef48f2aaf6c1ea6be0fb41 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/349c6afa7bef48f2aaf6c1ea6be0fb41 2024-11-21T01:45:16,086 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/3a27efbcb8684214a7f3eb2a5863e1e4 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/3a27efbcb8684214a7f3eb2a5863e1e4 2024-11-21T01:45:16,090 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/5fc0091a15574c9eacc04770de9b4a14 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/5fc0091a15574c9eacc04770de9b4a14 2024-11-21T01:45:16,091 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/72ed3722ce254cd399ea7b02264af5d2 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/72ed3722ce254cd399ea7b02264af5d2 2024-11-21T01:45:16,092 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/d4877e7f666446bdb6f333c0ec145c01 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/A/d4877e7f666446bdb6f333c0ec145c01 2024-11-21T01:45:16,094 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/12c0f63059784f3f8f3aa35584d91b35 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/12c0f63059784f3f8f3aa35584d91b35 2024-11-21T01:45:16,097 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/2bfa2856bed6427b8880e320d9664e4d to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/2bfa2856bed6427b8880e320d9664e4d 2024-11-21T01:45:16,098 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/a87aed0d0536484bb59edd0dd9e40a3f to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/a87aed0d0536484bb59edd0dd9e40a3f 2024-11-21T01:45:16,099 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/b68bdc81bff74c4c992b10d532cfd2f3 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/b68bdc81bff74c4c992b10d532cfd2f3 2024-11-21T01:45:16,100 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/e6fcfa4ee44a4f5a89f7c890b99a99e5 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/e6fcfa4ee44a4f5a89f7c890b99a99e5 2024-11-21T01:45:16,101 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/f619511b445c4c9d8834447b58c25fa4 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/B/f619511b445c4c9d8834447b58c25fa4 2024-11-21T01:45:16,103 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/3ce4c2a90c284693a0ff3b51ab6c00cb to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/3ce4c2a90c284693a0ff3b51ab6c00cb 2024-11-21T01:45:16,104 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/51181d0663984f36a2e460dee7dbd8f8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/51181d0663984f36a2e460dee7dbd8f8 2024-11-21T01:45:16,106 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/5dbc2b8a47e144c798e8b98fa06e2359 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/5dbc2b8a47e144c798e8b98fa06e2359 2024-11-21T01:45:16,107 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/6b5debbc0bde4531b46ce2204e6f1261 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/6b5debbc0bde4531b46ce2204e6f1261 2024-11-21T01:45:16,108 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/af93d7703bee4f818eab670896742f39 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/af93d7703bee4f818eab670896742f39 2024-11-21T01:45:16,109 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/d6ecf5bf1df943d9845f3039c77a360e to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/C/d6ecf5bf1df943d9845f3039c77a360e 2024-11-21T01:45:16,113 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/recovered.edits/771.seqid to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175/recovered.edits/771.seqid 2024-11-21T01:45:16,114 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/70b50c956032fda6af9bbc7628072175 2024-11-21T01:45:16,114 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T01:45:16,117 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=98, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:45:16,120 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T01:45:16,123 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T01:45:16,124 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=98, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:45:16,124 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T01:45:16,125 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732153516124"}]},"ts":"9223372036854775807"} 2024-11-21T01:45:16,127 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T01:45:16,127 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 70b50c956032fda6af9bbc7628072175, NAME => 'TestAcidGuarantees,,1732153491485.70b50c956032fda6af9bbc7628072175.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T01:45:16,127 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T01:45:16,127 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732153516127"}]},"ts":"9223372036854775807"} 2024-11-21T01:45:16,129 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T01:45:16,130 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=98, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:45:16,131 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 56 msec 2024-11-21T01:45:16,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-11-21T01:45:16,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T01:45:16,183 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:45:16,192 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252 (was 257), OpenFileDescriptor=461 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=590 (was 633), ProcessCount=11 (was 11), AvailableMemoryMB=2489 (was 3173) 2024-11-21T01:45:16,200 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=590, ProcessCount=11, AvailableMemoryMB=2489 2024-11-21T01:45:16,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T01:45:16,202 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T01:45:16,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T01:45:16,203 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T01:45:16,204 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:45:16,204 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 99 2024-11-21T01:45:16,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T01:45:16,204 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T01:45:16,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742178_1354 (size=1061) 2024-11-21T01:45:16,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T01:45:16,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T01:45:16,612 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d 2024-11-21T01:45:16,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742179_1355 (size=53) 2024-11-21T01:45:16,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T01:45:17,021 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:45:17,021 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8b49a551695b0705f38348b99dc083b8, disabling compactions & flushes 2024-11-21T01:45:17,021 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:17,021 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:17,021 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. after waiting 0 ms 2024-11-21T01:45:17,021 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:17,021 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:17,021 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8b49a551695b0705f38348b99dc083b8: Waiting for close lock at 1732153517021Disabling compacts and flushes for region at 1732153517021Disabling writes for close at 1732153517021Writing region close event to WAL at 1732153517021Closed at 1732153517021 2024-11-21T01:45:17,022 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T01:45:17,022 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732153517022"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732153517022"}]},"ts":"1732153517022"} 2024-11-21T01:45:17,023 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T01:45:17,024 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T01:45:17,024 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732153517024"}]},"ts":"1732153517024"} 2024-11-21T01:45:17,025 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T01:45:17,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b49a551695b0705f38348b99dc083b8, ASSIGN}] 2024-11-21T01:45:17,026 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b49a551695b0705f38348b99dc083b8, ASSIGN 2024-11-21T01:45:17,027 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b49a551695b0705f38348b99dc083b8, ASSIGN; state=OFFLINE, location=b93ada835ef8,41129,1732153434359; forceNewPlan=false, retain=false 2024-11-21T01:45:17,177 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=8b49a551695b0705f38348b99dc083b8, regionState=OPENING, regionLocation=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:17,178 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b49a551695b0705f38348b99dc083b8, ASSIGN because future has completed 2024-11-21T01:45:17,179 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359}] 2024-11-21T01:45:17,333 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:17,333 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(7752): Opening region: {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} 2024-11-21T01:45:17,333 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:17,333 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:45:17,333 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(7794): checking encryption for 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:17,333 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(7797): checking classloading for 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:17,334 INFO [StoreOpener-8b49a551695b0705f38348b99dc083b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:17,335 INFO [StoreOpener-8b49a551695b0705f38348b99dc083b8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T01:45:17,336 INFO [StoreOpener-8b49a551695b0705f38348b99dc083b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b49a551695b0705f38348b99dc083b8 columnFamilyName A 2024-11-21T01:45:17,336 DEBUG [StoreOpener-8b49a551695b0705f38348b99dc083b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:45:17,336 INFO [StoreOpener-8b49a551695b0705f38348b99dc083b8-1 {}] regionserver.HStore(327): Store=8b49a551695b0705f38348b99dc083b8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:45:17,336 INFO [StoreOpener-8b49a551695b0705f38348b99dc083b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:17,337 INFO [StoreOpener-8b49a551695b0705f38348b99dc083b8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T01:45:17,337 INFO [StoreOpener-8b49a551695b0705f38348b99dc083b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b49a551695b0705f38348b99dc083b8 columnFamilyName B 2024-11-21T01:45:17,337 DEBUG [StoreOpener-8b49a551695b0705f38348b99dc083b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:45:17,338 INFO [StoreOpener-8b49a551695b0705f38348b99dc083b8-1 {}] regionserver.HStore(327): Store=8b49a551695b0705f38348b99dc083b8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:45:17,338 INFO [StoreOpener-8b49a551695b0705f38348b99dc083b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:17,338 INFO [StoreOpener-8b49a551695b0705f38348b99dc083b8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T01:45:17,338 INFO [StoreOpener-8b49a551695b0705f38348b99dc083b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b49a551695b0705f38348b99dc083b8 columnFamilyName C 2024-11-21T01:45:17,338 DEBUG [StoreOpener-8b49a551695b0705f38348b99dc083b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:45:17,339 INFO [StoreOpener-8b49a551695b0705f38348b99dc083b8-1 {}] regionserver.HStore(327): Store=8b49a551695b0705f38348b99dc083b8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:45:17,339 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:17,339 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1038): replaying wal for 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:17,340 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:17,340 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:17,341 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1048): stopping wal replay for 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:17,341 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1060): Cleaning up temporary data for 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:17,341 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T01:45:17,342 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1093): writing seq id for 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:17,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T01:45:17,344 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T01:45:17,344 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1114): Opened 8b49a551695b0705f38348b99dc083b8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68766629, jitterRate=0.02470262348651886}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T01:45:17,344 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:17,345 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1006): Region open journal for 8b49a551695b0705f38348b99dc083b8: Running coprocessor pre-open hook at 1732153517333Writing region info on filesystem at 1732153517333Initializing all the Stores at 1732153517334 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732153517334Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732153517334Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732153517334Cleaning up temporary data from old regions at 1732153517341 (+7 ms)Running coprocessor post-open hooks at 1732153517344 (+3 ms)Region opened successfully at 1732153517345 (+1 ms) 2024-11-21T01:45:17,345 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., pid=101, masterSystemTime=1732153517330 2024-11-21T01:45:17,347 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:17,347 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:17,348 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=8b49a551695b0705f38348b99dc083b8, regionState=OPEN, openSeqNum=2, regionLocation=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:17,349 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36607 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=b93ada835ef8,41129,1732153434359, table=TestAcidGuarantees, region=8b49a551695b0705f38348b99dc083b8. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-21T01:45:17,349 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=101, ppid=100, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 because future has completed 2024-11-21T01:45:17,354 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-11-21T01:45:17,354 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; OpenRegionProcedure 8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 in 171 msec 2024-11-21T01:45:17,356 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-21T01:45:17,356 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b49a551695b0705f38348b99dc083b8, ASSIGN in 328 msec 2024-11-21T01:45:17,356 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T01:45:17,357 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732153517356"}]},"ts":"1732153517356"} 2024-11-21T01:45:17,358 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T01:45:17,359 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T01:45:17,359 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1570 sec 2024-11-21T01:45:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T01:45:18,353 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:45:18,354 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d468cd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:18,354 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:45:18,355 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:45:18,356 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:45:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:45:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:45:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f8943f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:45:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:45:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:18,357 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56318, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:45:18,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cefaf4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:18,358 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:45:18,359 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:45:18,359 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:45:18,360 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57958, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:45:18,361 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T01:45:18,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is b93ada835ef8,36607,1732153433607 2024-11-21T01:45:18,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6a55a02c 2024-11-21T01:45:18,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T01:45:18,362 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56328, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T01:45:18,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T01:45:18,364 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T01:45:18,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=102, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-21T01:45:18,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742180_1356 (size=1097) 2024-11-21T01:45:18,775 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-21T01:45:18,775 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-21T01:45:18,778 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T01:45:18,780 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b49a551695b0705f38348b99dc083b8, REOPEN/MOVE}] 2024-11-21T01:45:18,780 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b49a551695b0705f38348b99dc083b8, REOPEN/MOVE 2024-11-21T01:45:18,781 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=104 updating hbase:meta row=8b49a551695b0705f38348b99dc083b8, regionState=CLOSING, regionLocation=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:18,782 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b49a551695b0705f38348b99dc083b8, REOPEN/MOVE because future has completed 2024-11-21T01:45:18,782 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T01:45:18,782 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359}] 2024-11-21T01:45:18,935 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] handler.UnassignRegionHandler(122): Close 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:18,935 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T01:45:18,935 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1722): Closing 8b49a551695b0705f38348b99dc083b8, disabling compactions & flushes 2024-11-21T01:45:18,935 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:18,935 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:18,935 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. after waiting 0 ms 2024-11-21T01:45:18,935 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:18,939 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T01:45:18,939 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:18,939 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1676): Region close journal for 8b49a551695b0705f38348b99dc083b8: Waiting for close lock at 1732153518935Running coprocessor pre-close hooks at 1732153518935Disabling compacts and flushes for region at 1732153518935Disabling writes for close at 1732153518935Writing region close event to WAL at 1732153518935Running coprocessor post-close hooks at 1732153518939 (+4 ms)Closed at 1732153518939 2024-11-21T01:45:18,939 WARN [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegionServer(3299): Not adding moved region record: 8b49a551695b0705f38348b99dc083b8 to self. 2024-11-21T01:45:18,941 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] handler.UnassignRegionHandler(157): Closed 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:18,942 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=104 updating hbase:meta row=8b49a551695b0705f38348b99dc083b8, regionState=CLOSED 2024-11-21T01:45:18,943 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=105, ppid=104, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 because future has completed 2024-11-21T01:45:18,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-11-21T01:45:18,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; CloseRegionProcedure 8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 in 162 msec 2024-11-21T01:45:18,946 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b49a551695b0705f38348b99dc083b8, REOPEN/MOVE; state=CLOSED, location=b93ada835ef8,41129,1732153434359; forceNewPlan=false, retain=true 2024-11-21T01:45:19,096 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=104 updating hbase:meta row=8b49a551695b0705f38348b99dc083b8, regionState=OPENING, regionLocation=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:19,098 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b49a551695b0705f38348b99dc083b8, REOPEN/MOVE because future has completed 2024-11-21T01:45:19,098 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=104, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359}] 2024-11-21T01:45:19,253 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:19,253 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7752): Opening region: {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} 2024-11-21T01:45:19,254 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:19,254 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:45:19,254 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7794): checking encryption for 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:19,254 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7797): checking classloading for 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:19,256 INFO [StoreOpener-8b49a551695b0705f38348b99dc083b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:19,257 INFO [StoreOpener-8b49a551695b0705f38348b99dc083b8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T01:45:19,257 INFO [StoreOpener-8b49a551695b0705f38348b99dc083b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b49a551695b0705f38348b99dc083b8 columnFamilyName A 2024-11-21T01:45:19,258 DEBUG [StoreOpener-8b49a551695b0705f38348b99dc083b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:45:19,259 INFO [StoreOpener-8b49a551695b0705f38348b99dc083b8-1 {}] regionserver.HStore(327): Store=8b49a551695b0705f38348b99dc083b8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:45:19,259 INFO [StoreOpener-8b49a551695b0705f38348b99dc083b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:19,260 INFO [StoreOpener-8b49a551695b0705f38348b99dc083b8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T01:45:19,260 INFO [StoreOpener-8b49a551695b0705f38348b99dc083b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b49a551695b0705f38348b99dc083b8 columnFamilyName B 2024-11-21T01:45:19,260 DEBUG [StoreOpener-8b49a551695b0705f38348b99dc083b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:45:19,260 INFO [StoreOpener-8b49a551695b0705f38348b99dc083b8-1 {}] regionserver.HStore(327): Store=8b49a551695b0705f38348b99dc083b8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:45:19,260 INFO [StoreOpener-8b49a551695b0705f38348b99dc083b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:19,261 INFO [StoreOpener-8b49a551695b0705f38348b99dc083b8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T01:45:19,261 INFO [StoreOpener-8b49a551695b0705f38348b99dc083b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b49a551695b0705f38348b99dc083b8 columnFamilyName C 2024-11-21T01:45:19,261 DEBUG [StoreOpener-8b49a551695b0705f38348b99dc083b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:45:19,261 INFO [StoreOpener-8b49a551695b0705f38348b99dc083b8-1 {}] regionserver.HStore(327): Store=8b49a551695b0705f38348b99dc083b8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:45:19,261 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:19,261 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1038): replaying wal for 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:19,262 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:19,263 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:19,264 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1048): stopping wal replay for 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:19,264 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1060): Cleaning up temporary data for 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:19,264 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T01:45:19,266 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1093): writing seq id for 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:19,267 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1114): Opened 8b49a551695b0705f38348b99dc083b8; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62879769, jitterRate=-0.06301842629909515}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T01:45:19,267 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:19,267 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1006): Region open journal for 8b49a551695b0705f38348b99dc083b8: Running coprocessor pre-open hook at 1732153519254Writing region info on filesystem at 1732153519254Initializing all the Stores at 1732153519255 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732153519255Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732153519256 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732153519256Cleaning up temporary data from old regions at 1732153519264 (+8 ms)Running coprocessor post-open hooks at 1732153519267 (+3 ms)Region opened successfully at 1732153519267 2024-11-21T01:45:19,268 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., pid=106, masterSystemTime=1732153519250 2024-11-21T01:45:19,270 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:19,270 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:19,270 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=104 updating hbase:meta row=8b49a551695b0705f38348b99dc083b8, regionState=OPEN, openSeqNum=5, regionLocation=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:19,272 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=106, ppid=104, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 because future has completed 2024-11-21T01:45:19,274 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=104 2024-11-21T01:45:19,274 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=104, state=SUCCESS, hasLock=false; OpenRegionProcedure 8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 in 175 msec 2024-11-21T01:45:19,276 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-21T01:45:19,276 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b49a551695b0705f38348b99dc083b8, REOPEN/MOVE in 494 msec 2024-11-21T01:45:19,278 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-11-21T01:45:19,278 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 500 msec 2024-11-21T01:45:19,280 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 915 msec 2024-11-21T01:45:19,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=102 2024-11-21T01:45:19,281 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:45:19,282 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aba547e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:19,282 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:45:19,282 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:45:19,283 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:45:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:45:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:45:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b8a8dde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:45:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:45:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:19,285 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56342, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:45:19,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13ee4ff0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:19,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@440285d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:19,286 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:45:19,286 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:45:19,286 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:45:19,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:45:19,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:45:19,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3980bacb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:19,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:45:19,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:45:19,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:19,288 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:45:19,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c29227c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:19,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41c1e738, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:19,289 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:45:19,289 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:45:19,290 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:45:19,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:45:19,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:45:19,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24d0ac03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:19,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:45:19,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:45:19,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:19,291 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56388, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:45:19,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62a0a20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:19,292 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@508ab67b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:19,292 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:45:19,292 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:45:19,292 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:45:19,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:45:19,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:45:19,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9c3594, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:19,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:45:19,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:45:19,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:19,294 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56406, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:45:19,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@645626c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:19,295 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9d261f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:19,295 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:45:19,295 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:45:19,296 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:45:19,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:45:19,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:45:19,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37368c42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:19,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:45:19,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:45:19,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:19,297 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56420, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:45:19,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bfebc85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:19,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2853677b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:19,299 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:45:19,299 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:45:19,299 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:45:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:45:19,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:45:19,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13ab1d43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:19,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:45:19,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:45:19,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:19,301 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56434, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:45:19,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c9b62d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:19,302 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73d51c80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:19,302 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:45:19,302 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:45:19,303 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:45:19,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:45:19,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:45:19,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c4bbd71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:19,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:45:19,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:45:19,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:19,304 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56460, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:45:19,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2058de3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:19,305 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ff4aaae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:19,305 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:45:19,305 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:45:19,306 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:45:19,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:45:19,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:45:19,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48585838, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:19,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:45:19,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:45:19,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:19,307 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56472, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:45:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@713a6a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:19,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f15f223, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:19,308 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:45:19,308 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:45:19,309 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:45:19,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:45:19,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:45:19,310 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@feb579e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:19,310 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:45:19,310 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:45:19,310 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:19,311 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56488, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:45:19,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e8af9df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:19,312 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16b220e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:19,312 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:45:19,312 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:45:19,313 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:45:19,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:45:19,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:45:19,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5be2635c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:19,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:45:19,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:45:19,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:19,314 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56502, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:45:19,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@272f0ee6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:19,315 DEBUG [Thread-1627 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:45:19,315 DEBUG [Thread-1612 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:45:19,315 DEBUG [Thread-1618 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:45:19,316 DEBUG [Thread-1621 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:45:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:45:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:45:19,316 DEBUG [Thread-1610 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:45:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:45:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:45:19,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:45:19,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:45:19,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:45:19,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:45:19,318 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:45:19,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:45:19,318 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58006, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:45:19,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:45:19,319 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:45:19,319 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:45:19,320 DEBUG [Thread-1616 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:45:19,320 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:45:19,321 DEBUG [Thread-1616 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:45:19,321 DEBUG [Thread-1616 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:45:19,322 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58028, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:45:19,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5] 2024-11-21T01:45:19,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5] 2024-11-21T01:45:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5] 2024-11-21T01:45:19,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5] 2024-11-21T01:45:19,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5] 2024-11-21T01:45:19,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5] 2024-11-21T01:45:19,327 DEBUG [Thread-1625 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:45:19,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:45:19,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:45:19,329 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:45:19,329 DEBUG [Thread-1629 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:45:19,330 DEBUG [Thread-1629 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:45:19,330 DEBUG [Thread-1629 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:45:19,331 DEBUG [Thread-1614 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:45:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:45:19,332 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58042, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:45:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:45:19,332 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:45:19,333 DEBUG [Thread-1623 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:45:19,334 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58054, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:45:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5] 2024-11-21T01:45:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:45:19,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-21T01:45:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:45:19,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T01:45:19,336 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58060, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:45:19,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5] 2024-11-21T01:45:19,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5] 2024-11-21T01:45:19,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5] 2024-11-21T01:45:19,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b49a551695b0705f38348b99dc083b8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T01:45:19,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:19,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=A 2024-11-21T01:45:19,346 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:45:19,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:19,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=B 2024-11-21T01:45:19,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:19,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=C 2024-11-21T01:45:19,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:19,347 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:45:19,347 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:45:19,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ecb167b7b0b743b4aff95e2c51e47a07_8b49a551695b0705f38348b99dc083b8 is 50, key is test_row_0/A:col10/1732153519331/Put/seqid=0 2024-11-21T01:45:19,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742181_1357 (size=12154) 2024-11-21T01:45:19,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732153519384, max timestamp=1732153519404Num uniques 859; 2024-11-21T01:45:19,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T01:45:19,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732153519384, max timestamp=1732153519404Num uniques 859; 2024-11-21T01:45:19,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T01:45:19,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732153519384, max timestamp=1732153519404Num uniques 859; 2024-11-21T01:45:19,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9748, suffix heap size=117064,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=213 2024-11-21T01:45:19,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T01:45:19,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T01:45:19,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:19,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:19,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:19,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:19,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:19,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:19,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:19,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153579507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:19,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:19,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:19,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:19,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:19,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153579509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:19,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:19,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153579511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:19,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:19,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153579513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:19,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:19,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:19,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:19,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:19,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:19,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:19,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58018 deadline: 1732153579515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:19,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:19,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:19,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:19,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:19,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:19,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:19,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:19,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:19,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153579623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:19,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:19,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:19,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:19,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:19,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153579633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:19,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:19,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153579633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:19,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:19,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:19,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:19,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:19,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58018 deadline: 1732153579634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:19,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:19,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:19,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:19,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:19,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:19,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153579634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:19,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:19,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:19,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:19,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:19,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:19,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T01:45:19,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T01:45:19,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:19,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:19,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:19,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:19,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:19,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:19,793 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:45:19,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T01:45:19,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:19,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:19,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:19,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:19,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:19,809 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ecb167b7b0b743b4aff95e2c51e47a07_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ecb167b7b0b743b4aff95e2c51e47a07_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:19,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:19,814 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/56f130e31c154eff977caf50c150fb57, store: [table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:19,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/56f130e31c154eff977caf50c150fb57 is 175, key is test_row_0/A:col10/1732153519331/Put/seqid=0 2024-11-21T01:45:19,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742182_1358 (size=30951) 2024-11-21T01:45:19,842 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/56f130e31c154eff977caf50c150fb57 2024-11-21T01:45:19,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:19,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153579845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:19,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:19,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:19,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:19,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:19,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153579853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:19,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:19,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153579853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:19,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:19,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58018 deadline: 1732153579854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:19,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:19,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153579858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:19,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/19dae5c8400b4dc085ce873ba4b748ee is 50, key is test_row_0/B:col10/1732153519331/Put/seqid=0 2024-11-21T01:45:19,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742183_1359 (size=9657) 2024-11-21T01:45:19,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/19dae5c8400b4dc085ce873ba4b748ee 2024-11-21T01:45:19,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/1b22e72aa13e451bbf7d12c3efd81ee9 is 50, key is test_row_0/C:col10/1732153519331/Put/seqid=0 2024-11-21T01:45:19,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742184_1360 (size=9657) 2024-11-21T01:45:19,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T01:45:19,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:19,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:19,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:19,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:19,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:19,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:19,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T01:45:20,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T01:45:20,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:20,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:20,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:20,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:20,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:20,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:20,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:20,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153580163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:20,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:20,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:20,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:20,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:20,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153580173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:20,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:20,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:20,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:20,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153580173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:20,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:20,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:20,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:20,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:20,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58018 deadline: 1732153580173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:20,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:20,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:20,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:20,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153580173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:20,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:20,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:20,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:20,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:20,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:20,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T01:45:20,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:20,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:20,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:20,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:20,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:20,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:20,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/1b22e72aa13e451bbf7d12c3efd81ee9 2024-11-21T01:45:20,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/56f130e31c154eff977caf50c150fb57 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/56f130e31c154eff977caf50c150fb57 2024-11-21T01:45:20,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/56f130e31c154eff977caf50c150fb57, entries=150, sequenceid=14, filesize=30.2 K 2024-11-21T01:45:20,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/19dae5c8400b4dc085ce873ba4b748ee as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/19dae5c8400b4dc085ce873ba4b748ee 2024-11-21T01:45:20,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/19dae5c8400b4dc085ce873ba4b748ee, entries=100, sequenceid=14, filesize=9.4 K 2024-11-21T01:45:20,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/1b22e72aa13e451bbf7d12c3efd81ee9 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/1b22e72aa13e451bbf7d12c3efd81ee9 2024-11-21T01:45:20,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/1b22e72aa13e451bbf7d12c3efd81ee9, entries=100, sequenceid=14, filesize=9.4 K 2024-11-21T01:45:20,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=138.83 KB/142164 for 8b49a551695b0705f38348b99dc083b8 in 1032ms, sequenceid=14, compaction requested=false 2024-11-21T01:45:20,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:20,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T01:45:20,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:20,424 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 8b49a551695b0705f38348b99dc083b8 3/3 column families, dataSize=138.83 KB heapSize=390.86 KB 2024-11-21T01:45:20,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=A 2024-11-21T01:45:20,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:45:20,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=B 2024-11-21T01:45:20,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:45:20,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=C 2024-11-21T01:45:20,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:45:20,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ffb6a5318b914c45bf48e59937d26088_8b49a551695b0705f38348b99dc083b8 is 50, key is test_row_0/A:col10/1732153519512/Put/seqid=0 2024-11-21T01:45:20,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742185_1361 (size=12154) 2024-11-21T01:45:20,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:45:20,439 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ffb6a5318b914c45bf48e59937d26088_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ffb6a5318b914c45bf48e59937d26088_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:20,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/cdb9f21452794da19c415df8c8fcdb31, store: [table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:20,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/cdb9f21452794da19c415df8c8fcdb31 is 175, key is test_row_0/A:col10/1732153519512/Put/seqid=0 2024-11-21T01:45:20,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742186_1362 (size=30955) 2024-11-21T01:45:20,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T01:45:20,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:20,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:20,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:20,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58018 deadline: 1732153580703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:20,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:20,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153580705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:20,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:20,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:20,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:20,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:20,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153580709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:20,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:20,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:20,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:20,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:20,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153580714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:20,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:20,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:20,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:20,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:20,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153580714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:20,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:20,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:20,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:20,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:20,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58018 deadline: 1732153580824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:20,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:20,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:20,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:20,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:20,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153580825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:20,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:20,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:20,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:20,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:20,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153580825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:20,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:20,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:20,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:20,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:20,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153580836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:20,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:20,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153580836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:20,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:20,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:20,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:20,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:20,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:20,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:20,846 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/cdb9f21452794da19c415df8c8fcdb31 2024-11-21T01:45:20,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/2b822ffaa486456fa3b6bdeb15fc0ed1 is 50, key is test_row_0/B:col10/1732153519512/Put/seqid=0 2024-11-21T01:45:20,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742187_1363 (size=12001) 2024-11-21T01:45:21,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:21,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153581043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:21,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:21,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58018 deadline: 1732153581043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:21,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:21,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:21,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:21,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:21,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:21,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:21,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153581053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:21,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:21,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:21,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153581053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:21,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:21,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:21,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:21,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:21,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153581053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:21,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:21,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:21,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:21,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:21,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:21,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:21,264 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/2b822ffaa486456fa3b6bdeb15fc0ed1 2024-11-21T01:45:21,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/3f0ac36298e14fa498db4bf2a7789abd is 50, key is test_row_0/C:col10/1732153519512/Put/seqid=0 2024-11-21T01:45:21,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742188_1364 (size=12001) 2024-11-21T01:45:21,340 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T01:45:21,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:21,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153581363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:21,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:21,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:21,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:21,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:21,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153581363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:21,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:21,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:21,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:21,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153581363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:21,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:21,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:21,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153581363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:21,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:21,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:21,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:21,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:21,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58018 deadline: 1732153581363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:21,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:21,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:21,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:21,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:21,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:21,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T01:45:21,676 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.91 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/3f0ac36298e14fa498db4bf2a7789abd 2024-11-21T01:45:21,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/cdb9f21452794da19c415df8c8fcdb31 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/cdb9f21452794da19c415df8c8fcdb31 2024-11-21T01:45:21,698 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/cdb9f21452794da19c415df8c8fcdb31, entries=150, sequenceid=50, filesize=30.2 K 2024-11-21T01:45:21,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/2b822ffaa486456fa3b6bdeb15fc0ed1 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/2b822ffaa486456fa3b6bdeb15fc0ed1 2024-11-21T01:45:21,706 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/2b822ffaa486456fa3b6bdeb15fc0ed1, entries=150, sequenceid=50, filesize=11.7 K 2024-11-21T01:45:21,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/3f0ac36298e14fa498db4bf2a7789abd as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/3f0ac36298e14fa498db4bf2a7789abd 2024-11-21T01:45:21,711 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/3f0ac36298e14fa498db4bf2a7789abd, entries=150, sequenceid=50, filesize=11.7 K 2024-11-21T01:45:21,713 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~138.83 KB/142164, heapSize ~390.81 KB/400192, currentSize=53.67 KB/54960 for 8b49a551695b0705f38348b99dc083b8 in 1289ms, sequenceid=50, compaction requested=false 2024-11-21T01:45:21,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:21,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:21,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-21T01:45:21,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-21T01:45:21,716 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-21T01:45:21,716 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3670 sec 2024-11-21T01:45:21,718 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.3880 sec 2024-11-21T01:45:21,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:21,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b49a551695b0705f38348b99dc083b8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T01:45:21,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=A 2024-11-21T01:45:21,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:21,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=B 2024-11-21T01:45:21,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:21,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=C 2024-11-21T01:45:21,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:21,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c2415cb6986e48799c2a5501639e3e7f_8b49a551695b0705f38348b99dc083b8 is 50, key is test_row_0/A:col10/1732153520711/Put/seqid=0 2024-11-21T01:45:21,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742189_1365 (size=14594) 2024-11-21T01:45:21,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153581927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:21,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153581928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:21,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153581929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:21,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:21,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:21,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:21,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:21,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:21,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:21,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:21,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:21,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:21,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153581929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:21,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:21,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:21,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:21,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:21,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58018 deadline: 1732153581931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:21,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:21,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:21,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:22,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153582053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:22,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153582053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:22,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153582053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:22,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58018 deadline: 1732153582053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:22,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:22,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153582053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:22,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:22,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153582273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:22,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:22,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:22,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153582274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:22,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:22,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:22,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58018 deadline: 1732153582274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:22,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:22,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:22,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:22,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:22,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:22,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153582274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:22,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:22,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153582274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:22,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:22,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:22,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:22,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:22,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:22,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:22,295 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:45:22,299 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c2415cb6986e48799c2a5501639e3e7f_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c2415cb6986e48799c2a5501639e3e7f_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:22,300 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/3244f0623d3643a89ab93c52e4f1c546, store: [table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:22,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/3244f0623d3643a89ab93c52e4f1c546 is 175, key is test_row_0/A:col10/1732153520711/Put/seqid=0 2024-11-21T01:45:22,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742190_1366 (size=39549) 2024-11-21T01:45:22,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153582582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:22,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153582583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:22,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:22,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:22,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:22,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:22,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153582583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:22,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:22,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:22,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:22,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153582583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:22,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:22,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:22,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:22,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58018 deadline: 1732153582583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:22,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:22,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:22,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:22,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:22,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:22,630 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T01:45:22,712 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/3244f0623d3643a89ab93c52e4f1c546 2024-11-21T01:45:22,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/2ba45ebb61f6466cb22baa7bfdd3836f is 50, key is test_row_0/B:col10/1732153520711/Put/seqid=0 2024-11-21T01:45:22,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742191_1367 (size=12001) 2024-11-21T01:45:23,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153583102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:23,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:23,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:23,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:23,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153583102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:23,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:23,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:23,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:23,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153583102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:23,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:23,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:23,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:23,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153583103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:23,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:23,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:23,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:23,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:23,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58018 deadline: 1732153583103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:23,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:23,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:23,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:23,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/2ba45ebb61f6466cb22baa7bfdd3836f 2024-11-21T01:45:23,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/6b8ec015a88344f2a6b3a85df6384360 is 50, key is test_row_0/C:col10/1732153520711/Put/seqid=0 2024-11-21T01:45:23,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742192_1368 (size=12001) 2024-11-21T01:45:23,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T01:45:23,506 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:45:23,506 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:45:23,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-21T01:45:23,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T01:45:23,509 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:45:23,510 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:45:23,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:45:23,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/6b8ec015a88344f2a6b3a85df6384360 2024-11-21T01:45:23,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/3244f0623d3643a89ab93c52e4f1c546 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/3244f0623d3643a89ab93c52e4f1c546 2024-11-21T01:45:23,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/3244f0623d3643a89ab93c52e4f1c546, entries=200, sequenceid=62, filesize=38.6 K 2024-11-21T01:45:23,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/2ba45ebb61f6466cb22baa7bfdd3836f as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/2ba45ebb61f6466cb22baa7bfdd3836f 2024-11-21T01:45:23,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/2ba45ebb61f6466cb22baa7bfdd3836f, entries=150, sequenceid=62, filesize=11.7 K 2024-11-21T01:45:23,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/6b8ec015a88344f2a6b3a85df6384360 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/6b8ec015a88344f2a6b3a85df6384360 2024-11-21T01:45:23,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/6b8ec015a88344f2a6b3a85df6384360, entries=150, sequenceid=62, filesize=11.7 K 2024-11-21T01:45:23,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8b49a551695b0705f38348b99dc083b8 in 1693ms, sequenceid=62, compaction requested=true 2024-11-21T01:45:23,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:23,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b49a551695b0705f38348b99dc083b8:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:45:23,574 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:23,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:23,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b49a551695b0705f38348b99dc083b8:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:45:23,576 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:23,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:23,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b49a551695b0705f38348b99dc083b8:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:45:23,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:23,578 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:23,578 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101455 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:23,578 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 8b49a551695b0705f38348b99dc083b8/B is initiating minor compaction (all files) 2024-11-21T01:45:23,578 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 8b49a551695b0705f38348b99dc083b8/A is initiating minor compaction (all files) 2024-11-21T01:45:23,578 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b49a551695b0705f38348b99dc083b8/A in TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:23,578 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b49a551695b0705f38348b99dc083b8/B in TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:23,578 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/56f130e31c154eff977caf50c150fb57, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/cdb9f21452794da19c415df8c8fcdb31, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/3244f0623d3643a89ab93c52e4f1c546] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp, totalSize=99.1 K 2024-11-21T01:45:23,578 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/19dae5c8400b4dc085ce873ba4b748ee, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/2b822ffaa486456fa3b6bdeb15fc0ed1, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/2ba45ebb61f6466cb22baa7bfdd3836f] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp, totalSize=32.9 K 2024-11-21T01:45:23,578 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:23,578 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. files: [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/56f130e31c154eff977caf50c150fb57, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/cdb9f21452794da19c415df8c8fcdb31, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/3244f0623d3643a89ab93c52e4f1c546] 2024-11-21T01:45:23,579 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 19dae5c8400b4dc085ce873ba4b748ee, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732153519331 2024-11-21T01:45:23,579 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56f130e31c154eff977caf50c150fb57, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732153519329 2024-11-21T01:45:23,579 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdb9f21452794da19c415df8c8fcdb31, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732153519449 2024-11-21T01:45:23,579 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3244f0623d3643a89ab93c52e4f1c546, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732153520705 2024-11-21T01:45:23,579 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b822ffaa486456fa3b6bdeb15fc0ed1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732153519449 2024-11-21T01:45:23,580 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ba45ebb61f6466cb22baa7bfdd3836f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732153520705 2024-11-21T01:45:23,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T01:45:23,618 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b49a551695b0705f38348b99dc083b8#B#compaction#318 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:23,618 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/57b49760443d4e198bee85b45b73b486 is 50, key is test_row_0/B:col10/1732153520711/Put/seqid=0 2024-11-21T01:45:23,621 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:23,646 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112110d782f61e0a40549e6415511f0dbd0b_8b49a551695b0705f38348b99dc083b8 store=[table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:23,648 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112110d782f61e0a40549e6415511f0dbd0b_8b49a551695b0705f38348b99dc083b8, store=[table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:23,649 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112110d782f61e0a40549e6415511f0dbd0b_8b49a551695b0705f38348b99dc083b8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:23,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-21T01:45:23,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:23,662 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 8b49a551695b0705f38348b99dc083b8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T01:45:23,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=A 2024-11-21T01:45:23,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:23,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=B 2024-11-21T01:45:23,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:23,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=C 2024-11-21T01:45:23,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:23,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742193_1369 (size=12104) 2024-11-21T01:45:23,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742194_1370 (size=4469) 2024-11-21T01:45:23,675 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/57b49760443d4e198bee85b45b73b486 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/57b49760443d4e198bee85b45b73b486 2024-11-21T01:45:23,675 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b49a551695b0705f38348b99dc083b8#A#compaction#319 average throughput is 0.45 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:23,676 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/3401b2ed2ff44b9faed5f9421e9049dd is 175, key is test_row_0/A:col10/1732153520711/Put/seqid=0 2024-11-21T01:45:23,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211e220aae9a7f4404839ddb3ec15be172_8b49a551695b0705f38348b99dc083b8 is 50, key is test_row_0/A:col10/1732153521927/Put/seqid=0 2024-11-21T01:45:23,687 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b49a551695b0705f38348b99dc083b8/B of 8b49a551695b0705f38348b99dc083b8 into 57b49760443d4e198bee85b45b73b486(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:23,687 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:23,687 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., storeName=8b49a551695b0705f38348b99dc083b8/B, priority=13, startTime=1732153523576; duration=0sec 2024-11-21T01:45:23,687 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:23,687 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b49a551695b0705f38348b99dc083b8:B 2024-11-21T01:45:23,687 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:23,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742195_1371 (size=31058) 2024-11-21T01:45:23,689 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:23,689 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 8b49a551695b0705f38348b99dc083b8/C is initiating minor compaction (all files) 2024-11-21T01:45:23,689 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b49a551695b0705f38348b99dc083b8/C in TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:23,689 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/1b22e72aa13e451bbf7d12c3efd81ee9, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/3f0ac36298e14fa498db4bf2a7789abd, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/6b8ec015a88344f2a6b3a85df6384360] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp, totalSize=32.9 K 2024-11-21T01:45:23,690 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b22e72aa13e451bbf7d12c3efd81ee9, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732153519331 2024-11-21T01:45:23,690 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f0ac36298e14fa498db4bf2a7789abd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732153519449 2024-11-21T01:45:23,690 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b8ec015a88344f2a6b3a85df6384360, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732153520705 2024-11-21T01:45:23,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742196_1372 (size=12154) 2024-11-21T01:45:23,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:45:23,709 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b49a551695b0705f38348b99dc083b8#C#compaction#321 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:23,710 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/ef8dff03592947a895f97d24421cb32d is 50, key is test_row_0/C:col10/1732153520711/Put/seqid=0 2024-11-21T01:45:23,711 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211e220aae9a7f4404839ddb3ec15be172_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211e220aae9a7f4404839ddb3ec15be172_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:23,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/28cfdc16f0574784a69bdad8e5a367f0, store: [table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:23,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/28cfdc16f0574784a69bdad8e5a367f0 is 175, key is test_row_0/A:col10/1732153521927/Put/seqid=0 2024-11-21T01:45:23,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742197_1373 (size=12104) 2024-11-21T01:45:23,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742198_1374 (size=30955) 2024-11-21T01:45:23,737 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/ef8dff03592947a895f97d24421cb32d as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/ef8dff03592947a895f97d24421cb32d 2024-11-21T01:45:23,742 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b49a551695b0705f38348b99dc083b8/C of 8b49a551695b0705f38348b99dc083b8 into ef8dff03592947a895f97d24421cb32d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:23,742 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:23,742 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., storeName=8b49a551695b0705f38348b99dc083b8/C, priority=13, startTime=1732153523576; duration=0sec 2024-11-21T01:45:23,742 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:23,742 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b49a551695b0705f38348b99dc083b8:C 2024-11-21T01:45:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T01:45:24,097 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/3401b2ed2ff44b9faed5f9421e9049dd as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/3401b2ed2ff44b9faed5f9421e9049dd 2024-11-21T01:45:24,101 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b49a551695b0705f38348b99dc083b8/A of 8b49a551695b0705f38348b99dc083b8 into 3401b2ed2ff44b9faed5f9421e9049dd(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:24,102 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:24,102 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., storeName=8b49a551695b0705f38348b99dc083b8/A, priority=13, startTime=1732153523574; duration=0sec 2024-11-21T01:45:24,102 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:24,102 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b49a551695b0705f38348b99dc083b8:A 2024-11-21T01:45:24,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:24,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:24,132 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/28cfdc16f0574784a69bdad8e5a367f0 2024-11-21T01:45:24,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T01:45:24,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/8c668b396aa64f22b0b67ed74f818d5e is 50, key is test_row_0/B:col10/1732153521927/Put/seqid=0 2024-11-21T01:45:24,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:24,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58018 deadline: 1732153584142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:24,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:24,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:24,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153584142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:24,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:24,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:24,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153584144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:24,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:24,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:24,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153584145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:24,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:24,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153584146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:24,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:24,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742199_1375 (size=12001) 2024-11-21T01:45:24,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:24,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58018 deadline: 1732153584264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:24,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:24,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:24,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153584264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:24,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:24,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:24,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153584264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:24,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:24,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153584264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:24,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153584264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:24,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153584484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:24,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58018 deadline: 1732153584484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:24,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:24,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153584484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:24,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:24,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:24,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153584484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:24,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:24,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:24,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153584484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:24,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:24,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:24,562 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/8c668b396aa64f22b0b67ed74f818d5e 2024-11-21T01:45:24,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/d407b42ca0ae462e9da3a2e3991e847e is 50, key is test_row_0/C:col10/1732153521927/Put/seqid=0 2024-11-21T01:45:24,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742200_1376 (size=12001) 2024-11-21T01:45:24,594 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/d407b42ca0ae462e9da3a2e3991e847e 2024-11-21T01:45:24,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/28cfdc16f0574784a69bdad8e5a367f0 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/28cfdc16f0574784a69bdad8e5a367f0 2024-11-21T01:45:24,609 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/28cfdc16f0574784a69bdad8e5a367f0, entries=150, sequenceid=82, filesize=30.2 K 2024-11-21T01:45:24,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/8c668b396aa64f22b0b67ed74f818d5e as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/8c668b396aa64f22b0b67ed74f818d5e 2024-11-21T01:45:24,616 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/8c668b396aa64f22b0b67ed74f818d5e, entries=150, sequenceid=82, filesize=11.7 K 2024-11-21T01:45:24,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/d407b42ca0ae462e9da3a2e3991e847e as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/d407b42ca0ae462e9da3a2e3991e847e 2024-11-21T01:45:24,621 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/d407b42ca0ae462e9da3a2e3991e847e, entries=150, sequenceid=82, filesize=11.7 K 2024-11-21T01:45:24,622 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 8b49a551695b0705f38348b99dc083b8 in 960ms, sequenceid=82, compaction requested=false 2024-11-21T01:45:24,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:24,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:24,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-21T01:45:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-21T01:45:24,626 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-21T01:45:24,626 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1140 sec 2024-11-21T01:45:24,628 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.1210 sec 2024-11-21T01:45:24,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T01:45:24,643 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:45:24,645 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:45:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-21T01:45:24,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T01:45:24,654 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:45:24,655 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:45:24,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:45:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T01:45:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:24,806 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b49a551695b0705f38348b99dc083b8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T01:45:24,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=A 2024-11-21T01:45:24,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:24,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=B 2024-11-21T01:45:24,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:24,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=C 2024-11-21T01:45:24,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:24,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T01:45:24,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:24,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:24,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:24,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:24,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:24,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:24,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112168f4102588934a65b6c73d6fc9c379e0_8b49a551695b0705f38348b99dc083b8 is 50, key is test_row_0/A:col10/1732153524143/Put/seqid=0 2024-11-21T01:45:24,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742201_1377 (size=14594) 2024-11-21T01:45:24,824 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:45:24,829 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112168f4102588934a65b6c73d6fc9c379e0_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112168f4102588934a65b6c73d6fc9c379e0_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:24,829 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/28aebc80325941b58257c041c21f3985, store: [table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:24,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/28aebc80325941b58257c041c21f3985 is 175, key is test_row_0/A:col10/1732153524143/Put/seqid=0 2024-11-21T01:45:24,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742202_1378 (size=39549) 2024-11-21T01:45:24,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153584843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:24,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:24,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153584844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:24,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:24,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153584845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:24,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:24,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153584847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:24,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:24,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:24,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58018 deadline: 1732153584849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:24,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:24,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T01:45:24,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:24,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153584953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:24,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:24,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:24,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:24,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:24,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:24,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:24,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:24,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:24,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153584964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T01:45:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:24,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153584973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:24,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153584973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:24,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58018 deadline: 1732153584973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:24,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:24,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:24,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:25,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T01:45:25,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:25,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:25,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:25,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:25,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:25,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:25,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:25,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153585173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:25,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:25,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:25,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:25,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:25,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153585184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:25,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:25,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58018 deadline: 1732153585192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:25,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:25,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:25,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:25,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:25,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153585193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:25,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:25,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:25,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:25,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:25,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153585193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:25,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:25,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:25,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:25,234 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/28aebc80325941b58257c041c21f3985 2024-11-21T01:45:25,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/048bb9d5ca74410a85fd4b66e506b69c is 50, key is test_row_0/B:col10/1732153524143/Put/seqid=0 2024-11-21T01:45:25,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742203_1379 (size=12001) 2024-11-21T01:45:25,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T01:45:25,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:25,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:25,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:25,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:25,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:25,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:25,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T01:45:25,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T01:45:25,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:25,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:25,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:25,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:25,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:25,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:25,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:25,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153585493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:25,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:25,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:25,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:25,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:25,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153585503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:25,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:25,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:25,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:25,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:25,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58018 deadline: 1732153585512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:25,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:25,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153585512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:25,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:25,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:25,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:25,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:25,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153585513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:25,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:25,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:25,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:25,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:25,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:25,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:25,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T01:45:25,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:25,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:25,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:25,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:25,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:25,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/048bb9d5ca74410a85fd4b66e506b69c 2024-11-21T01:45:25,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/72fe32ac086f4028a3aaf8ebd4733bd3 is 50, key is test_row_0/C:col10/1732153524143/Put/seqid=0 2024-11-21T01:45:25,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742204_1380 (size=12001) 2024-11-21T01:45:25,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T01:45:25,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:25,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:25,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:25,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:25,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:25,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:25,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T01:45:25,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T01:45:25,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:25,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:25,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:25,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:25,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:25,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:26,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:26,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153586013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:26,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:26,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:26,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:26,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:26,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153586022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:26,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:26,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153586033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:26,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:26,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153586033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:26,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:26,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58018 deadline: 1732153586033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:26,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:26,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:26,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:26,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T01:45:26,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:26,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:26,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:26,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:26,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:26,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:26,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/72fe32ac086f4028a3aaf8ebd4733bd3 2024-11-21T01:45:26,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/28aebc80325941b58257c041c21f3985 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/28aebc80325941b58257c041c21f3985 2024-11-21T01:45:26,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/28aebc80325941b58257c041c21f3985, entries=200, sequenceid=98, filesize=38.6 K 2024-11-21T01:45:26,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/048bb9d5ca74410a85fd4b66e506b69c as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/048bb9d5ca74410a85fd4b66e506b69c 2024-11-21T01:45:26,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/048bb9d5ca74410a85fd4b66e506b69c, entries=150, sequenceid=98, filesize=11.7 K 2024-11-21T01:45:26,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/72fe32ac086f4028a3aaf8ebd4733bd3 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/72fe32ac086f4028a3aaf8ebd4733bd3 2024-11-21T01:45:26,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/72fe32ac086f4028a3aaf8ebd4733bd3, entries=150, sequenceid=98, filesize=11.7 K 2024-11-21T01:45:26,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 8b49a551695b0705f38348b99dc083b8 in 1304ms, sequenceid=98, compaction requested=true 2024-11-21T01:45:26,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:26,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b49a551695b0705f38348b99dc083b8:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:45:26,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:26,109 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:26,109 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:26,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b49a551695b0705f38348b99dc083b8:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:45:26,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:26,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b49a551695b0705f38348b99dc083b8:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:45:26,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:26,110 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:26,110 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:26,110 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 8b49a551695b0705f38348b99dc083b8/A is initiating minor compaction (all files) 2024-11-21T01:45:26,110 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 8b49a551695b0705f38348b99dc083b8/B is initiating minor compaction (all files) 2024-11-21T01:45:26,111 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b49a551695b0705f38348b99dc083b8/A in TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:26,111 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b49a551695b0705f38348b99dc083b8/B in TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:26,111 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/3401b2ed2ff44b9faed5f9421e9049dd, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/28cfdc16f0574784a69bdad8e5a367f0, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/28aebc80325941b58257c041c21f3985] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp, totalSize=99.2 K 2024-11-21T01:45:26,111 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/57b49760443d4e198bee85b45b73b486, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/8c668b396aa64f22b0b67ed74f818d5e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/048bb9d5ca74410a85fd4b66e506b69c] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp, totalSize=35.3 K 2024-11-21T01:45:26,111 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:26,111 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. files: [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/3401b2ed2ff44b9faed5f9421e9049dd, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/28cfdc16f0574784a69bdad8e5a367f0, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/28aebc80325941b58257c041c21f3985] 2024-11-21T01:45:26,111 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 57b49760443d4e198bee85b45b73b486, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732153520705 2024-11-21T01:45:26,111 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3401b2ed2ff44b9faed5f9421e9049dd, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732153520705 2024-11-21T01:45:26,112 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c668b396aa64f22b0b67ed74f818d5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732153521922 2024-11-21T01:45:26,112 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28cfdc16f0574784a69bdad8e5a367f0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732153521922 2024-11-21T01:45:26,112 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 048bb9d5ca74410a85fd4b66e506b69c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732153524139 2024-11-21T01:45:26,112 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28aebc80325941b58257c041c21f3985, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732153524139 2024-11-21T01:45:26,122 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:26,124 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b49a551695b0705f38348b99dc083b8#B#compaction#327 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:26,124 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/e8968163e02b47d4baf819c17366c668 is 50, key is test_row_0/B:col10/1732153524143/Put/seqid=0 2024-11-21T01:45:26,128 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411211855c46016274910ac8646b5ee44ba62_8b49a551695b0705f38348b99dc083b8 store=[table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:26,130 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411211855c46016274910ac8646b5ee44ba62_8b49a551695b0705f38348b99dc083b8, store=[table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:26,130 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211855c46016274910ac8646b5ee44ba62_8b49a551695b0705f38348b99dc083b8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:26,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742205_1381 (size=12207) 2024-11-21T01:45:26,148 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/e8968163e02b47d4baf819c17366c668 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/e8968163e02b47d4baf819c17366c668 2024-11-21T01:45:26,155 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b49a551695b0705f38348b99dc083b8/B of 8b49a551695b0705f38348b99dc083b8 into e8968163e02b47d4baf819c17366c668(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:26,155 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:26,156 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., storeName=8b49a551695b0705f38348b99dc083b8/B, priority=13, startTime=1732153526109; duration=0sec 2024-11-21T01:45:26,156 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:26,156 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b49a551695b0705f38348b99dc083b8:B 2024-11-21T01:45:26,156 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:26,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742206_1382 (size=4469) 2024-11-21T01:45:26,158 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:26,158 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 8b49a551695b0705f38348b99dc083b8/C is initiating minor compaction (all files) 2024-11-21T01:45:26,159 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b49a551695b0705f38348b99dc083b8/C in TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:26,159 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/ef8dff03592947a895f97d24421cb32d, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/d407b42ca0ae462e9da3a2e3991e847e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/72fe32ac086f4028a3aaf8ebd4733bd3] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp, totalSize=35.3 K 2024-11-21T01:45:26,159 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting ef8dff03592947a895f97d24421cb32d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732153520705 2024-11-21T01:45:26,159 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting d407b42ca0ae462e9da3a2e3991e847e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732153521922 2024-11-21T01:45:26,160 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 72fe32ac086f4028a3aaf8ebd4733bd3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732153524139 2024-11-21T01:45:26,171 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b49a551695b0705f38348b99dc083b8#C#compaction#329 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:26,171 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/67749dbdfe8c4e129cc2dc678452a998 is 50, key is test_row_0/C:col10/1732153524143/Put/seqid=0 2024-11-21T01:45:26,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742207_1383 (size=12207) 2024-11-21T01:45:26,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T01:45:26,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:26,203 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 8b49a551695b0705f38348b99dc083b8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T01:45:26,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=A 2024-11-21T01:45:26,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:26,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=B 2024-11-21T01:45:26,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:26,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=C 2024-11-21T01:45:26,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:26,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112158ed9ceccddf4564b60873e27a1304b5_8b49a551695b0705f38348b99dc083b8 is 50, key is test_row_0/A:col10/1732153524845/Put/seqid=0 2024-11-21T01:45:26,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742208_1384 (size=12154) 2024-11-21T01:45:26,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:45:26,264 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112158ed9ceccddf4564b60873e27a1304b5_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112158ed9ceccddf4564b60873e27a1304b5_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:26,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/b203f497dc80421693bcfe0a50e47d27, store: [table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:26,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/b203f497dc80421693bcfe0a50e47d27 is 175, key is test_row_0/A:col10/1732153524845/Put/seqid=0 2024-11-21T01:45:26,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742209_1385 (size=30955) 2024-11-21T01:45:26,558 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b49a551695b0705f38348b99dc083b8#A#compaction#328 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:26,559 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/2d287b3d3d6641c28d6540fde96aea48 is 175, key is test_row_0/A:col10/1732153524143/Put/seqid=0 2024-11-21T01:45:26,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742210_1386 (size=31161) 2024-11-21T01:45:26,592 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/67749dbdfe8c4e129cc2dc678452a998 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/67749dbdfe8c4e129cc2dc678452a998 2024-11-21T01:45:26,596 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b49a551695b0705f38348b99dc083b8/C of 8b49a551695b0705f38348b99dc083b8 into 67749dbdfe8c4e129cc2dc678452a998(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:26,596 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:26,596 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., storeName=8b49a551695b0705f38348b99dc083b8/C, priority=13, startTime=1732153526110; duration=0sec 2024-11-21T01:45:26,596 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:26,596 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b49a551695b0705f38348b99dc083b8:C 2024-11-21T01:45:26,683 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/b203f497dc80421693bcfe0a50e47d27 2024-11-21T01:45:26,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/3e1897a7f5c34bf6a2217c01df566541 is 50, key is test_row_0/B:col10/1732153524845/Put/seqid=0 2024-11-21T01:45:26,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742211_1387 (size=12001) 2024-11-21T01:45:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T01:45:26,969 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/2d287b3d3d6641c28d6540fde96aea48 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/2d287b3d3d6641c28d6540fde96aea48 2024-11-21T01:45:26,973 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b49a551695b0705f38348b99dc083b8/A of 8b49a551695b0705f38348b99dc083b8 into 2d287b3d3d6641c28d6540fde96aea48(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:26,973 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:26,973 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., storeName=8b49a551695b0705f38348b99dc083b8/A, priority=13, startTime=1732153526109; duration=0sec 2024-11-21T01:45:26,974 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:26,974 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b49a551695b0705f38348b99dc083b8:A 2024-11-21T01:45:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:27,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:27,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153587062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:27,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:27,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58018 deadline: 1732153587063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:27,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153587064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:27,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:27,073 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b93ada835ef8,41129,1732153434359 for 'test_row_1' in 8b49a551695b0705f38348b99dc083b8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T01:45:27,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:27,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153587065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:27,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:27,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153587067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:27,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:27,105 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/3e1897a7f5c34bf6a2217c01df566541 2024-11-21T01:45:27,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/9843668da1c54b9b81abbe76f65b4d72 is 50, key is test_row_0/C:col10/1732153524845/Put/seqid=0 2024-11-21T01:45:27,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742212_1388 (size=12001) 2024-11-21T01:45:27,117 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/9843668da1c54b9b81abbe76f65b4d72 2024-11-21T01:45:27,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/b203f497dc80421693bcfe0a50e47d27 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/b203f497dc80421693bcfe0a50e47d27 2024-11-21T01:45:27,129 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/b203f497dc80421693bcfe0a50e47d27, entries=150, sequenceid=118, filesize=30.2 K 2024-11-21T01:45:27,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/3e1897a7f5c34bf6a2217c01df566541 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/3e1897a7f5c34bf6a2217c01df566541 2024-11-21T01:45:27,134 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/3e1897a7f5c34bf6a2217c01df566541, entries=150, sequenceid=118, filesize=11.7 K 2024-11-21T01:45:27,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/9843668da1c54b9b81abbe76f65b4d72 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/9843668da1c54b9b81abbe76f65b4d72 2024-11-21T01:45:27,138 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/9843668da1c54b9b81abbe76f65b4d72, entries=150, sequenceid=118, filesize=11.7 K 2024-11-21T01:45:27,139 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 8b49a551695b0705f38348b99dc083b8 in 936ms, sequenceid=118, compaction requested=false 2024-11-21T01:45:27,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:27,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:27,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-21T01:45:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-21T01:45:27,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-21T01:45:27,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4860 sec 2024-11-21T01:45:27,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.4990 sec 2024-11-21T01:45:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:27,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b49a551695b0705f38348b99dc083b8 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T01:45:27,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=A 2024-11-21T01:45:27,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:27,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=B 2024-11-21T01:45:27,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:27,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=C 2024-11-21T01:45:27,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:27,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216a42d998d6204ef5b6640c1992dfd700_8b49a551695b0705f38348b99dc083b8 is 50, key is test_row_0/A:col10/1732153527186/Put/seqid=0 2024-11-21T01:45:27,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742213_1389 (size=14744) 2024-11-21T01:45:27,207 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:45:27,210 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216a42d998d6204ef5b6640c1992dfd700_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216a42d998d6204ef5b6640c1992dfd700_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:27,211 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/582f75afa64a4c60970f5e09219b70f6, store: [table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:27,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/582f75afa64a4c60970f5e09219b70f6 is 175, key is test_row_0/A:col10/1732153527186/Put/seqid=0 2024-11-21T01:45:27,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742214_1390 (size=39699) 2024-11-21T01:45:27,221 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/582f75afa64a4c60970f5e09219b70f6 2024-11-21T01:45:27,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/283352b73604400198ca12e6e84db867 is 50, key is test_row_0/B:col10/1732153527186/Put/seqid=0 2024-11-21T01:45:27,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:27,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153587222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:27,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:27,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742215_1391 (size=12101) 2024-11-21T01:45:27,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:27,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153587240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:27,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:27,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153587241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:27,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:27,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153587243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:27,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:27,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:27,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153587353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:27,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:27,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:27,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153587364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153587364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:27,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:27,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153587364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:27,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:27,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:27,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:27,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:27,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153587584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:27,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:27,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153587593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:27,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:27,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153587593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:27,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:27,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:27,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153587593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:27,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:27,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:27,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/283352b73604400198ca12e6e84db867 2024-11-21T01:45:27,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/6b8cab8b2c3c48039bad60afbb36569a is 50, key is test_row_0/C:col10/1732153527186/Put/seqid=0 2024-11-21T01:45:27,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742216_1392 (size=12101) 2024-11-21T01:45:27,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/6b8cab8b2c3c48039bad60afbb36569a 2024-11-21T01:45:27,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/582f75afa64a4c60970f5e09219b70f6 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/582f75afa64a4c60970f5e09219b70f6 2024-11-21T01:45:27,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/582f75afa64a4c60970f5e09219b70f6, entries=200, sequenceid=134, filesize=38.8 K 2024-11-21T01:45:27,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/283352b73604400198ca12e6e84db867 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/283352b73604400198ca12e6e84db867 2024-11-21T01:45:27,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/283352b73604400198ca12e6e84db867, entries=150, sequenceid=134, filesize=11.8 K 2024-11-21T01:45:27,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/6b8cab8b2c3c48039bad60afbb36569a as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/6b8cab8b2c3c48039bad60afbb36569a 2024-11-21T01:45:27,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/6b8cab8b2c3c48039bad60afbb36569a, entries=150, sequenceid=134, filesize=11.8 K 2024-11-21T01:45:27,716 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 8b49a551695b0705f38348b99dc083b8 in 529ms, sequenceid=134, compaction requested=true 2024-11-21T01:45:27,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:27,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b49a551695b0705f38348b99dc083b8:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:45:27,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:27,716 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:27,716 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:27,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b49a551695b0705f38348b99dc083b8:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:45:27,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:27,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b49a551695b0705f38348b99dc083b8:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:45:27,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:27,719 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:27,719 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 8b49a551695b0705f38348b99dc083b8/B is initiating minor compaction (all files) 2024-11-21T01:45:27,719 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b49a551695b0705f38348b99dc083b8/B in TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:27,719 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/e8968163e02b47d4baf819c17366c668, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/3e1897a7f5c34bf6a2217c01df566541, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/283352b73604400198ca12e6e84db867] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp, totalSize=35.5 K 2024-11-21T01:45:27,719 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting e8968163e02b47d4baf819c17366c668, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732153524139 2024-11-21T01:45:27,719 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101815 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:27,719 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 8b49a551695b0705f38348b99dc083b8/A is initiating minor compaction (all files) 2024-11-21T01:45:27,719 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b49a551695b0705f38348b99dc083b8/A in TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:27,720 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/2d287b3d3d6641c28d6540fde96aea48, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/b203f497dc80421693bcfe0a50e47d27, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/582f75afa64a4c60970f5e09219b70f6] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp, totalSize=99.4 K 2024-11-21T01:45:27,720 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:27,720 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. files: [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/2d287b3d3d6641c28d6540fde96aea48, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/b203f497dc80421693bcfe0a50e47d27, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/582f75afa64a4c60970f5e09219b70f6] 2024-11-21T01:45:27,721 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e1897a7f5c34bf6a2217c01df566541, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732153524843 2024-11-21T01:45:27,721 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d287b3d3d6641c28d6540fde96aea48, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732153524139 2024-11-21T01:45:27,722 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 283352b73604400198ca12e6e84db867, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732153527037 2024-11-21T01:45:27,722 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting b203f497dc80421693bcfe0a50e47d27, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732153524843 2024-11-21T01:45:27,722 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 582f75afa64a4c60970f5e09219b70f6, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732153527037 2024-11-21T01:45:27,732 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:27,733 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b49a551695b0705f38348b99dc083b8#B#compaction#336 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:27,733 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/a23b19f7b229494d8a13f866e6100406 is 50, key is test_row_0/B:col10/1732153527186/Put/seqid=0 2024-11-21T01:45:27,734 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112141693ba929854b2bbb2820030fc4c1e4_8b49a551695b0705f38348b99dc083b8 store=[table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:27,736 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112141693ba929854b2bbb2820030fc4c1e4_8b49a551695b0705f38348b99dc083b8, store=[table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:27,736 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112141693ba929854b2bbb2820030fc4c1e4_8b49a551695b0705f38348b99dc083b8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:27,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742217_1393 (size=12409) 2024-11-21T01:45:27,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742218_1394 (size=4469) 2024-11-21T01:45:27,789 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b49a551695b0705f38348b99dc083b8#A#compaction#337 average throughput is 0.43 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:27,790 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/2186f7aba1c747529fa03da7c41466ab is 175, key is test_row_0/A:col10/1732153527186/Put/seqid=0 2024-11-21T01:45:27,791 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/a23b19f7b229494d8a13f866e6100406 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/a23b19f7b229494d8a13f866e6100406 2024-11-21T01:45:27,808 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b49a551695b0705f38348b99dc083b8/B of 8b49a551695b0705f38348b99dc083b8 into a23b19f7b229494d8a13f866e6100406(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:27,808 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:27,808 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., storeName=8b49a551695b0705f38348b99dc083b8/B, priority=13, startTime=1732153527716; duration=0sec 2024-11-21T01:45:27,808 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:27,808 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b49a551695b0705f38348b99dc083b8:B 2024-11-21T01:45:27,808 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:27,813 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:27,813 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 8b49a551695b0705f38348b99dc083b8/C is initiating minor compaction (all files) 2024-11-21T01:45:27,813 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b49a551695b0705f38348b99dc083b8/C in TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:27,813 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/67749dbdfe8c4e129cc2dc678452a998, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/9843668da1c54b9b81abbe76f65b4d72, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/6b8cab8b2c3c48039bad60afbb36569a] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp, totalSize=35.5 K 2024-11-21T01:45:27,813 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 67749dbdfe8c4e129cc2dc678452a998, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732153524139 2024-11-21T01:45:27,816 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 9843668da1c54b9b81abbe76f65b4d72, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732153524843 2024-11-21T01:45:27,819 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b8cab8b2c3c48039bad60afbb36569a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732153527037 2024-11-21T01:45:27,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742219_1395 (size=31363) 2024-11-21T01:45:27,860 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b49a551695b0705f38348b99dc083b8#C#compaction#338 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:27,860 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/afc33746fb5c416d8de036ae815bb438 is 50, key is test_row_0/C:col10/1732153527186/Put/seqid=0 2024-11-21T01:45:27,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742220_1396 (size=12409) 2024-11-21T01:45:27,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b49a551695b0705f38348b99dc083b8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T01:45:27,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=A 2024-11-21T01:45:27,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:27,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=B 2024-11-21T01:45:27,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:27,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=C 2024-11-21T01:45:27,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:27,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:27,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121eab8cf561b9d4e108a1feca826a4c0ca_8b49a551695b0705f38348b99dc083b8 is 50, key is test_row_0/A:col10/1732153527220/Put/seqid=0 2024-11-21T01:45:27,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742221_1397 (size=14794) 2024-11-21T01:45:27,937 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:45:27,945 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121eab8cf561b9d4e108a1feca826a4c0ca_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121eab8cf561b9d4e108a1feca826a4c0ca_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:27,946 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/9171802b7302428eb6ac2f647c18121b, store: [table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:27,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/9171802b7302428eb6ac2f647c18121b is 175, key is test_row_0/A:col10/1732153527220/Put/seqid=0 2024-11-21T01:45:27,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742222_1398 (size=39749) 2024-11-21T01:45:27,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153587939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:27,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153587940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:27,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:27,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:27,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153587941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:27,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153587943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:27,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:27,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:27,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:28,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153588063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:28,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:28,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153588063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:28,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:28,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:28,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:28,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:28,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153588063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:28,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:28,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:28,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:28,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:28,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:28,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153588073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:28,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:28,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:28,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:28,240 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/2186f7aba1c747529fa03da7c41466ab as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/2186f7aba1c747529fa03da7c41466ab 2024-11-21T01:45:28,244 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b49a551695b0705f38348b99dc083b8/A of 8b49a551695b0705f38348b99dc083b8 into 2186f7aba1c747529fa03da7c41466ab(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:28,244 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:28,245 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., storeName=8b49a551695b0705f38348b99dc083b8/A, priority=13, startTime=1732153527716; duration=0sec 2024-11-21T01:45:28,245 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:28,245 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b49a551695b0705f38348b99dc083b8:A 2024-11-21T01:45:28,278 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/afc33746fb5c416d8de036ae815bb438 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/afc33746fb5c416d8de036ae815bb438 2024-11-21T01:45:28,282 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b49a551695b0705f38348b99dc083b8/C of 8b49a551695b0705f38348b99dc083b8 into afc33746fb5c416d8de036ae815bb438(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:28,282 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:28,282 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., storeName=8b49a551695b0705f38348b99dc083b8/C, priority=13, startTime=1732153527716; duration=0sec 2024-11-21T01:45:28,282 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:28,282 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b49a551695b0705f38348b99dc083b8:C 2024-11-21T01:45:28,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:28,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153588272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:28,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:28,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:28,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153588272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:28,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:28,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153588273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:28,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:28,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:28,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:28,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:28,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:28,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:28,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:28,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:28,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:28,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153588293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:28,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:28,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:28,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:28,355 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/9171802b7302428eb6ac2f647c18121b 2024-11-21T01:45:28,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/82ad106da1df4d7d891706ac67f6fbcf is 50, key is test_row_0/B:col10/1732153527220/Put/seqid=0 2024-11-21T01:45:28,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742223_1399 (size=12151) 2024-11-21T01:45:28,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153588593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:28,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153588593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:28,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153588593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:28,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:28,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:28,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153588605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:28,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:28,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:28,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:28,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/82ad106da1df4d7d891706ac67f6fbcf 2024-11-21T01:45:28,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/e0f5fde9b98547eeb5ad2968dcb6a2af is 50, key is test_row_0/C:col10/1732153527220/Put/seqid=0 2024-11-21T01:45:28,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742224_1400 (size=12151) 2024-11-21T01:45:28,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T01:45:28,814 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:45:28,814 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:45:28,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-21T01:45:28,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T01:45:28,816 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:45:28,817 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:45:28,817 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:45:28,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T01:45:28,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-21T01:45:28,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:28,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:28,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:28,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:28,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:28,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:29,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58018 deadline: 1732153589092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:29,104 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b93ada835ef8,41129,1732153434359 for 'test_row_1' in 8b49a551695b0705f38348b99dc083b8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T01:45:29,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:29,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:29,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:29,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-21T01:45:29,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:29,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:29,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:29,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:29,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:29,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153589113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:29,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:29,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:29,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:29,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153589113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:29,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:29,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:29,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:29,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153589123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:29,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:29,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:29,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:29,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153589123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:29,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:29,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:29,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T01:45:29,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/e0f5fde9b98547eeb5ad2968dcb6a2af 2024-11-21T01:45:29,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/9171802b7302428eb6ac2f647c18121b as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/9171802b7302428eb6ac2f647c18121b 2024-11-21T01:45:29,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/9171802b7302428eb6ac2f647c18121b, entries=200, sequenceid=154, filesize=38.8 K 2024-11-21T01:45:29,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/82ad106da1df4d7d891706ac67f6fbcf as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/82ad106da1df4d7d891706ac67f6fbcf 2024-11-21T01:45:29,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/82ad106da1df4d7d891706ac67f6fbcf, entries=150, sequenceid=154, filesize=11.9 K 2024-11-21T01:45:29,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/e0f5fde9b98547eeb5ad2968dcb6a2af as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/e0f5fde9b98547eeb5ad2968dcb6a2af 2024-11-21T01:45:29,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/e0f5fde9b98547eeb5ad2968dcb6a2af, entries=150, sequenceid=154, filesize=11.9 K 2024-11-21T01:45:29,239 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 8b49a551695b0705f38348b99dc083b8 in 1329ms, sequenceid=154, compaction requested=false 2024-11-21T01:45:29,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:29,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-21T01:45:29,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:29,282 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 8b49a551695b0705f38348b99dc083b8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T01:45:29,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=A 2024-11-21T01:45:29,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:29,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=B 2024-11-21T01:45:29,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:29,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=C 2024-11-21T01:45:29,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:29,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411217c7187638b6f4a74b13b87ef2ed34c6c_8b49a551695b0705f38348b99dc083b8 is 50, key is test_row_0/A:col10/1732153527938/Put/seqid=0 2024-11-21T01:45:29,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742225_1401 (size=12304) 2024-11-21T01:45:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T01:45:29,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:45:29,765 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411217c7187638b6f4a74b13b87ef2ed34c6c_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217c7187638b6f4a74b13b87ef2ed34c6c_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:29,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/ff292309d21c4d699aaff07dacf4ec10, store: [table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:29,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/ff292309d21c4d699aaff07dacf4ec10 is 175, key is test_row_0/A:col10/1732153527938/Put/seqid=0 2024-11-21T01:45:29,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742226_1402 (size=31105) 2024-11-21T01:45:29,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T01:45:30,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:30,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:30,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:30,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153590194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:30,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:30,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:30,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:30,221 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/ff292309d21c4d699aaff07dacf4ec10 2024-11-21T01:45:30,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:30,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153590208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:30,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:30,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:30,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:30,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:30,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153590210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:30,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/8c3744a1f9824def94abfd75c26d0363 is 50, key is test_row_0/B:col10/1732153527938/Put/seqid=0 2024-11-21T01:45:30,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:30,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153590212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:30,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:30,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:30,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:30,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:30,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:30,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:30,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742227_1403 (size=12151) 2024-11-21T01:45:30,252 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/8c3744a1f9824def94abfd75c26d0363 2024-11-21T01:45:30,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/6f66b42c28f14fd4ad19701581135f34 is 50, key is test_row_0/C:col10/1732153527938/Put/seqid=0 2024-11-21T01:45:30,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742228_1404 (size=12151) 2024-11-21T01:45:30,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:30,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153590322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:30,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:30,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:30,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:30,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153590334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:30,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:30,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:30,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153590334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:30,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:30,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153590334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:30,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:30,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:30,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:30,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:30,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:30,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:30,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:30,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:30,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:30,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:30,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153590543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:30,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:30,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:30,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:30,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153590543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:30,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:30,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:30,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:30,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:30,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:30,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153590544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:30,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:30,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:30,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153590544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:30,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:30,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:30,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:30,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:30,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:30,675 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/6f66b42c28f14fd4ad19701581135f34 2024-11-21T01:45:30,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/ff292309d21c4d699aaff07dacf4ec10 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/ff292309d21c4d699aaff07dacf4ec10 2024-11-21T01:45:30,685 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/ff292309d21c4d699aaff07dacf4ec10, entries=150, sequenceid=169, filesize=30.4 K 2024-11-21T01:45:30,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/8c3744a1f9824def94abfd75c26d0363 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/8c3744a1f9824def94abfd75c26d0363 2024-11-21T01:45:30,704 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/8c3744a1f9824def94abfd75c26d0363, entries=150, sequenceid=169, filesize=11.9 K 2024-11-21T01:45:30,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/6f66b42c28f14fd4ad19701581135f34 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/6f66b42c28f14fd4ad19701581135f34 2024-11-21T01:45:30,711 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/6f66b42c28f14fd4ad19701581135f34, entries=150, sequenceid=169, filesize=11.9 K 2024-11-21T01:45:30,712 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 8b49a551695b0705f38348b99dc083b8 in 1430ms, sequenceid=169, compaction requested=true 2024-11-21T01:45:30,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:30,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:30,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-21T01:45:30,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-21T01:45:30,716 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-21T01:45:30,716 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8970 sec 2024-11-21T01:45:30,718 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.9030 sec 2024-11-21T01:45:30,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b49a551695b0705f38348b99dc083b8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T01:45:30,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:30,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=A 2024-11-21T01:45:30,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:30,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=B 2024-11-21T01:45:30,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:30,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=C 2024-11-21T01:45:30,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:30,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219fc01a591ebe4cb09058581a17862423_8b49a551695b0705f38348b99dc083b8 is 50, key is test_row_0/A:col10/1732153530863/Put/seqid=0 2024-11-21T01:45:30,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742229_1405 (size=14794) 2024-11-21T01:45:30,894 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:45:30,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:30,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153590887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:30,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:30,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153590889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:30,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:30,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:30,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:30,914 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219fc01a591ebe4cb09058581a17862423_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219fc01a591ebe4cb09058581a17862423_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:30,915 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/874db9e4775244fc982a874eb51fed47, store: [table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:30,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/874db9e4775244fc982a874eb51fed47 is 175, key is test_row_0/A:col10/1732153530863/Put/seqid=0 2024-11-21T01:45:30,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:30,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153590908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:30,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:30,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:30,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:30,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:30,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153590911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:30,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:30,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:30,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:30,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742230_1406 (size=39749) 2024-11-21T01:45:30,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T01:45:30,965 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:45:30,966 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:45:30,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-21T01:45:30,969 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:45:30,970 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:45:30,970 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:45:30,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T01:45:31,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153591013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:31,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:31,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153591023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:31,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:31,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153591033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:31,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:31,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153591033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:31,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T01:45:31,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-21T01:45:31,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:31,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:31,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:31,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:31,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:31,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:31,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153591233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:31,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:31,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153591242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153591242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:31,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153591243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:31,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-21T01:45:31,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:31,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:31,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:31,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:31,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:31,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T01:45:31,362 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=189, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/874db9e4775244fc982a874eb51fed47 2024-11-21T01:45:31,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/a00034a26e884793a27a5e6fec667e1a is 50, key is test_row_0/B:col10/1732153530863/Put/seqid=0 2024-11-21T01:45:31,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742231_1407 (size=12151) 2024-11-21T01:45:31,414 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/a00034a26e884793a27a5e6fec667e1a 2024-11-21T01:45:31,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/8cd012d06bda4884991ecb1fcf846226 is 50, key is test_row_0/C:col10/1732153530863/Put/seqid=0 2024-11-21T01:45:31,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-21T01:45:31,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:31,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:31,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:31,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:31,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:31,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742232_1408 (size=12151) 2024-11-21T01:45:31,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/8cd012d06bda4884991ecb1fcf846226 2024-11-21T01:45:31,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/874db9e4775244fc982a874eb51fed47 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/874db9e4775244fc982a874eb51fed47 2024-11-21T01:45:31,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/874db9e4775244fc982a874eb51fed47, entries=200, sequenceid=189, filesize=38.8 K 2024-11-21T01:45:31,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/a00034a26e884793a27a5e6fec667e1a as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/a00034a26e884793a27a5e6fec667e1a 2024-11-21T01:45:31,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/a00034a26e884793a27a5e6fec667e1a, entries=150, sequenceid=189, filesize=11.9 K 2024-11-21T01:45:31,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/8cd012d06bda4884991ecb1fcf846226 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/8cd012d06bda4884991ecb1fcf846226 2024-11-21T01:45:31,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/8cd012d06bda4884991ecb1fcf846226, entries=150, sequenceid=189, filesize=11.9 K 2024-11-21T01:45:31,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 8b49a551695b0705f38348b99dc083b8 in 634ms, sequenceid=189, compaction requested=true 2024-11-21T01:45:31,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:31,500 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:45:31,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b49a551695b0705f38348b99dc083b8:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:45:31,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:31,500 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:45:31,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b49a551695b0705f38348b99dc083b8:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:45:31,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:31,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b49a551695b0705f38348b99dc083b8:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:45:31,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:31,502 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:45:31,503 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 8b49a551695b0705f38348b99dc083b8/B is initiating minor compaction (all files) 2024-11-21T01:45:31,503 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b49a551695b0705f38348b99dc083b8/B in TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:31,503 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/a23b19f7b229494d8a13f866e6100406, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/82ad106da1df4d7d891706ac67f6fbcf, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/8c3744a1f9824def94abfd75c26d0363, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/a00034a26e884793a27a5e6fec667e1a] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp, totalSize=47.7 K 2024-11-21T01:45:31,503 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141966 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:45:31,503 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 8b49a551695b0705f38348b99dc083b8/A is initiating minor compaction (all files) 2024-11-21T01:45:31,503 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b49a551695b0705f38348b99dc083b8/A in TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:31,503 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting a23b19f7b229494d8a13f866e6100406, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732153527037 2024-11-21T01:45:31,503 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/2186f7aba1c747529fa03da7c41466ab, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/9171802b7302428eb6ac2f647c18121b, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/ff292309d21c4d699aaff07dacf4ec10, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/874db9e4775244fc982a874eb51fed47] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp, totalSize=138.6 K 2024-11-21T01:45:31,503 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:31,503 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. files: [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/2186f7aba1c747529fa03da7c41466ab, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/9171802b7302428eb6ac2f647c18121b, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/ff292309d21c4d699aaff07dacf4ec10, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/874db9e4775244fc982a874eb51fed47] 2024-11-21T01:45:31,504 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 82ad106da1df4d7d891706ac67f6fbcf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732153527220 2024-11-21T01:45:31,504 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2186f7aba1c747529fa03da7c41466ab, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732153527037 2024-11-21T01:45:31,504 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c3744a1f9824def94abfd75c26d0363, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732153527938 2024-11-21T01:45:31,504 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9171802b7302428eb6ac2f647c18121b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732153527220 2024-11-21T01:45:31,505 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting a00034a26e884793a27a5e6fec667e1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732153530208 2024-11-21T01:45:31,505 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff292309d21c4d699aaff07dacf4ec10, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732153527938 2024-11-21T01:45:31,508 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 874db9e4775244fc982a874eb51fed47, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732153530191 2024-11-21T01:45:31,514 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:31,518 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b49a551695b0705f38348b99dc083b8#B#compaction#349 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:31,519 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/bcd2e179c0a04fc194e938d38a8c6fb6 is 50, key is test_row_0/B:col10/1732153530863/Put/seqid=0 2024-11-21T01:45:31,536 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121a91ce233e1d14dcaaf2a59c1bb5e879b_8b49a551695b0705f38348b99dc083b8 store=[table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:31,539 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121a91ce233e1d14dcaaf2a59c1bb5e879b_8b49a551695b0705f38348b99dc083b8, store=[table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:31,539 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a91ce233e1d14dcaaf2a59c1bb5e879b_8b49a551695b0705f38348b99dc083b8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:31,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742233_1409 (size=12595) 2024-11-21T01:45:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:31,547 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b49a551695b0705f38348b99dc083b8 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T01:45:31,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=A 2024-11-21T01:45:31,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:31,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=B 2024-11-21T01:45:31,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:31,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=C 2024-11-21T01:45:31,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:31,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212b704011824541f5a4010f7315788849_8b49a551695b0705f38348b99dc083b8 is 50, key is test_row_0/A:col10/1732153531546/Put/seqid=0 2024-11-21T01:45:31,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742234_1410 (size=4469) 2024-11-21T01:45:31,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742235_1411 (size=14794) 2024-11-21T01:45:31,572 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:45:31,576 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b49a551695b0705f38348b99dc083b8#A#compaction#348 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:31,577 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/95abd6145e1d478da742a548bcb39b1b is 175, key is test_row_0/A:col10/1732153530863/Put/seqid=0 2024-11-21T01:45:31,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-21T01:45:31,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:31,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:31,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:31,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:31,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:31,586 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212b704011824541f5a4010f7315788849_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212b704011824541f5a4010f7315788849_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:31,587 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/6c796a97dd594cbeb33371476fe7b29c, store: [table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:31,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/6c796a97dd594cbeb33371476fe7b29c is 175, key is test_row_0/A:col10/1732153531546/Put/seqid=0 2024-11-21T01:45:31,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742236_1412 (size=31549) 2024-11-21T01:45:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T01:45:31,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742237_1413 (size=39749) 2024-11-21T01:45:31,606 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/95abd6145e1d478da742a548bcb39b1b as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/95abd6145e1d478da742a548bcb39b1b 2024-11-21T01:45:31,610 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=203, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/6c796a97dd594cbeb33371476fe7b29c 2024-11-21T01:45:31,610 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8b49a551695b0705f38348b99dc083b8/A of 8b49a551695b0705f38348b99dc083b8 into 95abd6145e1d478da742a548bcb39b1b(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:31,610 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:31,610 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., storeName=8b49a551695b0705f38348b99dc083b8/A, priority=12, startTime=1732153531500; duration=0sec 2024-11-21T01:45:31,611 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:31,611 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b49a551695b0705f38348b99dc083b8:A 2024-11-21T01:45:31,611 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:45:31,612 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:45:31,612 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 8b49a551695b0705f38348b99dc083b8/C is initiating minor compaction (all files) 2024-11-21T01:45:31,612 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b49a551695b0705f38348b99dc083b8/C in TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:31,612 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/afc33746fb5c416d8de036ae815bb438, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/e0f5fde9b98547eeb5ad2968dcb6a2af, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/6f66b42c28f14fd4ad19701581135f34, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/8cd012d06bda4884991ecb1fcf846226] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp, totalSize=47.7 K 2024-11-21T01:45:31,613 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting afc33746fb5c416d8de036ae815bb438, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732153527037 2024-11-21T01:45:31,614 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0f5fde9b98547eeb5ad2968dcb6a2af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732153527220 2024-11-21T01:45:31,614 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f66b42c28f14fd4ad19701581135f34, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732153527938 2024-11-21T01:45:31,615 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cd012d06bda4884991ecb1fcf846226, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732153530208 2024-11-21T01:45:31,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/bc8c2aa527814cb1aefad5bda4566648 is 50, key is test_row_0/B:col10/1732153531546/Put/seqid=0 2024-11-21T01:45:31,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742238_1414 (size=12151) 2024-11-21T01:45:31,625 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b49a551695b0705f38348b99dc083b8#C#compaction#352 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:31,627 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/2e66ae67eb6541aa9a3a2d40ec6f92e6 is 50, key is test_row_0/C:col10/1732153530863/Put/seqid=0 2024-11-21T01:45:31,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/bc8c2aa527814cb1aefad5bda4566648 2024-11-21T01:45:31,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742239_1415 (size=12595) 2024-11-21T01:45:31,638 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/2e66ae67eb6541aa9a3a2d40ec6f92e6 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/2e66ae67eb6541aa9a3a2d40ec6f92e6 2024-11-21T01:45:31,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/51104c3b09f94e58994467b6e27e73d2 is 50, key is test_row_0/C:col10/1732153531546/Put/seqid=0 2024-11-21T01:45:31,648 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8b49a551695b0705f38348b99dc083b8/C of 8b49a551695b0705f38348b99dc083b8 into 2e66ae67eb6541aa9a3a2d40ec6f92e6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:31,648 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:31,648 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., storeName=8b49a551695b0705f38348b99dc083b8/C, priority=12, startTime=1732153531501; duration=0sec 2024-11-21T01:45:31,648 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:31,648 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b49a551695b0705f38348b99dc083b8:C 2024-11-21T01:45:31,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153591628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:31,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:31,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153591632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:31,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:31,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153591632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:31,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:31,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153591634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:31,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:31,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:31,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742240_1416 (size=12151) 2024-11-21T01:45:31,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-21T01:45:31,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:31,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:31,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:31,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:31,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:31,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:31,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:31,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153591753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:31,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:31,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153591753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:31,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:31,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:31,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:31,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153591762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:31,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153591762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:31,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-21T01:45:31,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:31,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:31,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:31,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:31,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:31,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:31,961 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/bcd2e179c0a04fc194e938d38a8c6fb6 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/bcd2e179c0a04fc194e938d38a8c6fb6 2024-11-21T01:45:31,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:31,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153591963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:31,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:31,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:31,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153591964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:31,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:31,972 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8b49a551695b0705f38348b99dc083b8/B of 8b49a551695b0705f38348b99dc083b8 into bcd2e179c0a04fc194e938d38a8c6fb6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:31,972 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:31,972 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., storeName=8b49a551695b0705f38348b99dc083b8/B, priority=12, startTime=1732153531500; duration=0sec 2024-11-21T01:45:31,972 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:31,972 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b49a551695b0705f38348b99dc083b8:B 2024-11-21T01:45:31,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:31,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153591982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:31,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:31,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153591983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:31,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:31,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:31,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:32,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-21T01:45:32,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:32,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:32,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:32,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:32,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:32,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:32,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/51104c3b09f94e58994467b6e27e73d2 2024-11-21T01:45:32,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/6c796a97dd594cbeb33371476fe7b29c as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/6c796a97dd594cbeb33371476fe7b29c 2024-11-21T01:45:32,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/6c796a97dd594cbeb33371476fe7b29c, entries=200, sequenceid=203, filesize=38.8 K 2024-11-21T01:45:32,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/bc8c2aa527814cb1aefad5bda4566648 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/bc8c2aa527814cb1aefad5bda4566648 2024-11-21T01:45:32,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/bc8c2aa527814cb1aefad5bda4566648, entries=150, sequenceid=203, filesize=11.9 K 2024-11-21T01:45:32,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/51104c3b09f94e58994467b6e27e73d2 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/51104c3b09f94e58994467b6e27e73d2 2024-11-21T01:45:32,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/51104c3b09f94e58994467b6e27e73d2, entries=150, sequenceid=203, filesize=11.9 K 2024-11-21T01:45:32,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 8b49a551695b0705f38348b99dc083b8 in 531ms, sequenceid=203, compaction requested=false 2024-11-21T01:45:32,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:32,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T01:45:32,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-21T01:45:32,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:32,201 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 8b49a551695b0705f38348b99dc083b8 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-21T01:45:32,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=A 2024-11-21T01:45:32,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:32,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=B 2024-11-21T01:45:32,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:32,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=C 2024-11-21T01:45:32,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:32,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211982ae7cb8b24eb5bbb36b026b987df3_8b49a551695b0705f38348b99dc083b8 is 50, key is test_row_0/A:col10/1732153531605/Put/seqid=0 2024-11-21T01:45:32,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742241_1417 (size=12304) 2024-11-21T01:45:32,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:45:32,230 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211982ae7cb8b24eb5bbb36b026b987df3_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211982ae7cb8b24eb5bbb36b026b987df3_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:32,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/f18d43b3ff1f44cf8931e8990f8e3242, store: [table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:32,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/f18d43b3ff1f44cf8931e8990f8e3242 is 175, key is test_row_0/A:col10/1732153531605/Put/seqid=0 2024-11-21T01:45:32,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742242_1418 (size=31105) 2024-11-21T01:45:32,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:32,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:32,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:32,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153592313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:32,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:32,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:32,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:32,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:32,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153592327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:32,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:32,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:32,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:32,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:32,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153592330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:32,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:32,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:32,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:32,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:32,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153592331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:32,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:32,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:32,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:32,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:32,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153592443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:32,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:32,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:32,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:32,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:32,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153592443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:32,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:32,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153592453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:32,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:32,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:32,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:32,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:32,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:32,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:32,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:32,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153592457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:32,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:32,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:32,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:32,641 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=224, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/f18d43b3ff1f44cf8931e8990f8e3242 2024-11-21T01:45:32,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/19bfef38f4c24487b2d936c29a4bce9f is 50, key is test_row_0/B:col10/1732153531605/Put/seqid=0 2024-11-21T01:45:32,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742243_1419 (size=12151) 2024-11-21T01:45:32,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:32,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153592663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:32,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:32,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:32,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:32,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:32,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153592663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:32,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:32,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153592663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:32,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:32,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:32,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:32,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:32,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:32,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:32,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:32,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153592672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:32,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:32,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:32,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:32,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:32,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153592983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:32,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:32,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:32,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:32,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153592993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:32,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:32,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:32,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153592993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:32,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:32,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:32,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:32,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:32,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:32,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153592993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:32,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:32,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:32,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:33,053 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/19bfef38f4c24487b2d936c29a4bce9f 2024-11-21T01:45:33,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/dc464de248fc41f59548ddeeb886892d is 50, key is test_row_0/C:col10/1732153531605/Put/seqid=0 2024-11-21T01:45:33,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742244_1420 (size=12151) 2024-11-21T01:45:33,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T01:45:33,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:33,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58018 deadline: 1732153593133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:33,145 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b93ada835ef8,41129,1732153434359 for 'test_row_1' in 8b49a551695b0705f38348b99dc083b8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8296 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T01:45:33,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:33,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:33,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:33,494 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/dc464de248fc41f59548ddeeb886892d 2024-11-21T01:45:33,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/f18d43b3ff1f44cf8931e8990f8e3242 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/f18d43b3ff1f44cf8931e8990f8e3242 2024-11-21T01:45:33,503 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/f18d43b3ff1f44cf8931e8990f8e3242, entries=150, sequenceid=224, filesize=30.4 K 2024-11-21T01:45:33,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/19bfef38f4c24487b2d936c29a4bce9f as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/19bfef38f4c24487b2d936c29a4bce9f 2024-11-21T01:45:33,507 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/19bfef38f4c24487b2d936c29a4bce9f, entries=150, sequenceid=224, filesize=11.9 K 2024-11-21T01:45:33,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/dc464de248fc41f59548ddeeb886892d as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/dc464de248fc41f59548ddeeb886892d 2024-11-21T01:45:33,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:33,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153593503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:33,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:33,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153593503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:33,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:33,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153593503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:33,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:33,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153593503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:33,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:33,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:33,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:33,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:33,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:33,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:33,512 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/dc464de248fc41f59548ddeeb886892d, entries=150, sequenceid=224, filesize=11.9 K 2024-11-21T01:45:33,512 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 8b49a551695b0705f38348b99dc083b8 in 1311ms, sequenceid=224, compaction requested=true 2024-11-21T01:45:33,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:33,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:33,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-21T01:45:33,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-21T01:45:33,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-21T01:45:33,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5440 sec 2024-11-21T01:45:33,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.5510 sec 2024-11-21T01:45:34,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:34,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b49a551695b0705f38348b99dc083b8 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-21T01:45:34,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=A 2024-11-21T01:45:34,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:34,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=B 2024-11-21T01:45:34,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:34,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=C 2024-11-21T01:45:34,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:34,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216a31aa9fade349eea24b0574855cb98b_8b49a551695b0705f38348b99dc083b8 is 50, key is test_row_0/A:col10/1732153532330/Put/seqid=0 2024-11-21T01:45:34,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742245_1421 (size=14794) 2024-11-21T01:45:34,554 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:45:34,561 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216a31aa9fade349eea24b0574855cb98b_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216a31aa9fade349eea24b0574855cb98b_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:34,562 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/95dbda7768ee4df689309f1470b51e90, store: [table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:34,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/95dbda7768ee4df689309f1470b51e90 is 175, key is test_row_0/A:col10/1732153532330/Put/seqid=0 2024-11-21T01:45:34,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742246_1422 (size=39749) 2024-11-21T01:45:34,567 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/95dbda7768ee4df689309f1470b51e90 2024-11-21T01:45:34,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/28c6ee3971d44173ae3b2af4533a6144 is 50, key is test_row_0/B:col10/1732153532330/Put/seqid=0 2024-11-21T01:45:34,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742247_1423 (size=12151) 2024-11-21T01:45:34,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/28c6ee3971d44173ae3b2af4533a6144 2024-11-21T01:45:34,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/28832761adf54e1babe97b32cdac4167 is 50, key is test_row_0/C:col10/1732153532330/Put/seqid=0 2024-11-21T01:45:34,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:34,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153594581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:34,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:34,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:34,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:34,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153594581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:34,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:34,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:34,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:34,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153594583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:34,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:34,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:34,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:34,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:34,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153594598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:34,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:34,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:34,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:34,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742248_1424 (size=12151) 2024-11-21T01:45:34,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/28832761adf54e1babe97b32cdac4167 2024-11-21T01:45:34,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/95dbda7768ee4df689309f1470b51e90 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/95dbda7768ee4df689309f1470b51e90 2024-11-21T01:45:34,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/95dbda7768ee4df689309f1470b51e90, entries=200, sequenceid=239, filesize=38.8 K 2024-11-21T01:45:34,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/28c6ee3971d44173ae3b2af4533a6144 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/28c6ee3971d44173ae3b2af4533a6144 2024-11-21T01:45:34,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/28c6ee3971d44173ae3b2af4533a6144, entries=150, sequenceid=239, filesize=11.9 K 2024-11-21T01:45:34,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/28832761adf54e1babe97b32cdac4167 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/28832761adf54e1babe97b32cdac4167 2024-11-21T01:45:34,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/28832761adf54e1babe97b32cdac4167, entries=150, sequenceid=239, filesize=11.9 K 2024-11-21T01:45:34,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 8b49a551695b0705f38348b99dc083b8 in 119ms, sequenceid=239, compaction requested=true 2024-11-21T01:45:34,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:34,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b49a551695b0705f38348b99dc083b8:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:45:34,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:34,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b49a551695b0705f38348b99dc083b8:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:45:34,640 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:45:34,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:34,640 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:45:34,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b49a551695b0705f38348b99dc083b8:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:45:34,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:34,641 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142152 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:45:34,641 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 8b49a551695b0705f38348b99dc083b8/A is initiating minor compaction (all files) 2024-11-21T01:45:34,641 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b49a551695b0705f38348b99dc083b8/A in TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:34,641 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/95abd6145e1d478da742a548bcb39b1b, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/6c796a97dd594cbeb33371476fe7b29c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/f18d43b3ff1f44cf8931e8990f8e3242, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/95dbda7768ee4df689309f1470b51e90] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp, totalSize=138.8 K 2024-11-21T01:45:34,641 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:34,642 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. files: [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/95abd6145e1d478da742a548bcb39b1b, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/6c796a97dd594cbeb33371476fe7b29c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/f18d43b3ff1f44cf8931e8990f8e3242, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/95dbda7768ee4df689309f1470b51e90] 2024-11-21T01:45:34,642 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:45:34,642 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 8b49a551695b0705f38348b99dc083b8/B is initiating minor compaction (all files) 2024-11-21T01:45:34,642 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b49a551695b0705f38348b99dc083b8/B in TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:34,642 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/bcd2e179c0a04fc194e938d38a8c6fb6, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/bc8c2aa527814cb1aefad5bda4566648, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/19bfef38f4c24487b2d936c29a4bce9f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/28c6ee3971d44173ae3b2af4533a6144] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp, totalSize=47.9 K 2024-11-21T01:45:34,642 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95abd6145e1d478da742a548bcb39b1b, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732153530208 2024-11-21T01:45:34,643 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c796a97dd594cbeb33371476fe7b29c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732153530882 2024-11-21T01:45:34,643 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting bcd2e179c0a04fc194e938d38a8c6fb6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732153530208 2024-11-21T01:45:34,643 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting f18d43b3ff1f44cf8931e8990f8e3242, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732153531605 2024-11-21T01:45:34,643 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95dbda7768ee4df689309f1470b51e90, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732153532307 2024-11-21T01:45:34,643 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting bc8c2aa527814cb1aefad5bda4566648, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732153530882 2024-11-21T01:45:34,646 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 19bfef38f4c24487b2d936c29a4bce9f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732153531605 2024-11-21T01:45:34,647 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 28c6ee3971d44173ae3b2af4533a6144, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732153532325 2024-11-21T01:45:34,657 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:34,680 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b49a551695b0705f38348b99dc083b8#B#compaction#361 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:34,682 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112132cb75fbb4c947059b270fd9495297a8_8b49a551695b0705f38348b99dc083b8 store=[table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:34,684 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112132cb75fbb4c947059b270fd9495297a8_8b49a551695b0705f38348b99dc083b8, store=[table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:34,684 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112132cb75fbb4c947059b270fd9495297a8_8b49a551695b0705f38348b99dc083b8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:34,692 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/1ae59e794c9c4de0abfa7be41bc7f332 is 50, key is test_row_0/B:col10/1732153532330/Put/seqid=0 2024-11-21T01:45:34,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742249_1425 (size=4469) 2024-11-21T01:45:34,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742250_1426 (size=12731) 2024-11-21T01:45:34,719 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/1ae59e794c9c4de0abfa7be41bc7f332 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/1ae59e794c9c4de0abfa7be41bc7f332 2024-11-21T01:45:34,724 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8b49a551695b0705f38348b99dc083b8/B of 8b49a551695b0705f38348b99dc083b8 into 1ae59e794c9c4de0abfa7be41bc7f332(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:34,724 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:34,724 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., storeName=8b49a551695b0705f38348b99dc083b8/B, priority=12, startTime=1732153534640; duration=0sec 2024-11-21T01:45:34,725 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:34,725 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b49a551695b0705f38348b99dc083b8:B 2024-11-21T01:45:34,725 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:45:34,726 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:45:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:34,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b49a551695b0705f38348b99dc083b8 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-21T01:45:34,726 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 8b49a551695b0705f38348b99dc083b8/C is initiating minor compaction (all files) 2024-11-21T01:45:34,726 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b49a551695b0705f38348b99dc083b8/C in TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:34,727 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/2e66ae67eb6541aa9a3a2d40ec6f92e6, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/51104c3b09f94e58994467b6e27e73d2, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/dc464de248fc41f59548ddeeb886892d, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/28832761adf54e1babe97b32cdac4167] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp, totalSize=47.9 K 2024-11-21T01:45:34,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=A 2024-11-21T01:45:34,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:34,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=B 2024-11-21T01:45:34,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:34,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=C 2024-11-21T01:45:34,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:34,727 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e66ae67eb6541aa9a3a2d40ec6f92e6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732153530208 2024-11-21T01:45:34,727 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 51104c3b09f94e58994467b6e27e73d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732153530882 2024-11-21T01:45:34,728 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting dc464de248fc41f59548ddeeb886892d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732153531605 2024-11-21T01:45:34,728 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 28832761adf54e1babe97b32cdac4167, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732153532325 2024-11-21T01:45:34,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a2cff75a73b4465ca7bd1f62d99f2dcd_8b49a551695b0705f38348b99dc083b8 is 50, key is test_row_0/A:col10/1732153534578/Put/seqid=0 2024-11-21T01:45:34,755 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b49a551695b0705f38348b99dc083b8#C#compaction#363 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:34,755 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/62a0aeaee8f0448ab551aadcc785df63 is 50, key is test_row_0/C:col10/1732153532330/Put/seqid=0 2024-11-21T01:45:34,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742251_1427 (size=14844) 2024-11-21T01:45:34,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742252_1428 (size=12731) 2024-11-21T01:45:34,779 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/62a0aeaee8f0448ab551aadcc785df63 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/62a0aeaee8f0448ab551aadcc785df63 2024-11-21T01:45:34,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153594767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:34,784 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8b49a551695b0705f38348b99dc083b8/C of 8b49a551695b0705f38348b99dc083b8 into 62a0aeaee8f0448ab551aadcc785df63(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:34,784 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:34,784 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., storeName=8b49a551695b0705f38348b99dc083b8/C, priority=12, startTime=1732153534640; duration=0sec 2024-11-21T01:45:34,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:34,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:34,784 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:34,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:34,784 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b49a551695b0705f38348b99dc083b8:C 2024-11-21T01:45:34,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153594770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:34,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:34,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:34,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:34,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153594774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:34,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153594774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:34,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:34,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:34,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:34,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:34,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:34,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:34,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:34,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153594893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:34,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:34,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153594893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:34,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:34,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153594893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:34,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:34,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153594893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:34,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:34,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:34,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:35,105 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b49a551695b0705f38348b99dc083b8#A#compaction#360 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:35,106 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/d3ea01ecb3e34fe1a896bf860ddd60ed is 175, key is test_row_0/A:col10/1732153532330/Put/seqid=0 2024-11-21T01:45:35,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:35,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153595113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:35,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:35,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153595113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:35,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:35,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:35,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:35,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:35,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:35,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:35,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153595113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:35,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:35,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:35,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:35,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153595113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:35,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:35,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:35,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:35,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742253_1429 (size=31685) 2024-11-21T01:45:35,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T01:45:35,133 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:45:35,133 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:45:35,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-21T01:45:35,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T01:45:35,136 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:45:35,136 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:45:35,137 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:45:35,174 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:45:35,185 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a2cff75a73b4465ca7bd1f62d99f2dcd_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a2cff75a73b4465ca7bd1f62d99f2dcd_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:35,186 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/654ffdd2377141b5aeb75658c9950098, store: [table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:35,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/654ffdd2377141b5aeb75658c9950098 is 175, key is test_row_0/A:col10/1732153534578/Put/seqid=0 2024-11-21T01:45:35,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742254_1430 (size=39799) 2024-11-21T01:45:35,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T01:45:35,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-21T01:45:35,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:35,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:35,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:35,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:35,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:35,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:35,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:35,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153595433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:35,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:35,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:35,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:35,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:35,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153595433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:35,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:35,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153595433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:35,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:35,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153595434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:35,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-21T01:45:35,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:35,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:35,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:35,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:35,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:35,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:35,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T01:45:35,537 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/d3ea01ecb3e34fe1a896bf860ddd60ed as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/d3ea01ecb3e34fe1a896bf860ddd60ed 2024-11-21T01:45:35,543 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8b49a551695b0705f38348b99dc083b8/A of 8b49a551695b0705f38348b99dc083b8 into d3ea01ecb3e34fe1a896bf860ddd60ed(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:35,543 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:35,543 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., storeName=8b49a551695b0705f38348b99dc083b8/A, priority=12, startTime=1732153534639; duration=0sec 2024-11-21T01:45:35,543 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:35,543 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b49a551695b0705f38348b99dc083b8:A 2024-11-21T01:45:35,595 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/654ffdd2377141b5aeb75658c9950098 2024-11-21T01:45:35,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/e2d5f5dcb66440d091c22186ee287987 is 50, key is test_row_0/B:col10/1732153534578/Put/seqid=0 2024-11-21T01:45:35,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-21T01:45:35,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:35,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:35,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:35,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:35,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:35,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:35,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742255_1431 (size=12201) 2024-11-21T01:45:35,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/e2d5f5dcb66440d091c22186ee287987 2024-11-21T01:45:35,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/14db3c0008514778a94e5e71545b7921 is 50, key is test_row_0/C:col10/1732153534578/Put/seqid=0 2024-11-21T01:45:35,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742256_1432 (size=12201) 2024-11-21T01:45:35,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/14db3c0008514778a94e5e71545b7921 2024-11-21T01:45:35,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/654ffdd2377141b5aeb75658c9950098 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/654ffdd2377141b5aeb75658c9950098 2024-11-21T01:45:35,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/654ffdd2377141b5aeb75658c9950098, entries=200, sequenceid=258, filesize=38.9 K 2024-11-21T01:45:35,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/e2d5f5dcb66440d091c22186ee287987 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/e2d5f5dcb66440d091c22186ee287987 2024-11-21T01:45:35,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/e2d5f5dcb66440d091c22186ee287987, entries=150, sequenceid=258, filesize=11.9 K 2024-11-21T01:45:35,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/14db3c0008514778a94e5e71545b7921 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/14db3c0008514778a94e5e71545b7921 2024-11-21T01:45:35,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/14db3c0008514778a94e5e71545b7921, entries=150, sequenceid=258, filesize=11.9 K 2024-11-21T01:45:35,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 8b49a551695b0705f38348b99dc083b8 in 957ms, sequenceid=258, compaction requested=false 2024-11-21T01:45:35,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:35,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-21T01:45:35,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:35,763 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 8b49a551695b0705f38348b99dc083b8 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T01:45:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T01:45:35,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=A 2024-11-21T01:45:35,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:35,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=B 2024-11-21T01:45:35,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:35,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=C 2024-11-21T01:45:35,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:35,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b539400cb2ac405281a36c057eb1b386_8b49a551695b0705f38348b99dc083b8 is 50, key is test_row_0/A:col10/1732153534768/Put/seqid=0 2024-11-21T01:45:35,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742257_1433 (size=12454) 2024-11-21T01:45:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:35,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:36,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:36,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153595990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:36,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:36,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:36,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:36,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153595991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:36,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:36,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:36,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:36,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:36,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153595993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:36,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:36,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:36,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:36,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:36,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153596006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:36,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:36,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:36,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:36,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153596113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:36,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153596113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:36,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:36,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153596123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:36,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:36,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:36,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:36,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:36,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153596132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:36,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:36,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:36,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:36,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:45:36,206 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b539400cb2ac405281a36c057eb1b386_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b539400cb2ac405281a36c057eb1b386_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:36,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/10ee9b35fba0465c9e0cb7b8dd2dba0e, store: [table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:36,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/10ee9b35fba0465c9e0cb7b8dd2dba0e is 175, key is test_row_0/A:col10/1732153534768/Put/seqid=0 2024-11-21T01:45:36,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742258_1434 (size=31255) 2024-11-21T01:45:36,217 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/10ee9b35fba0465c9e0cb7b8dd2dba0e 2024-11-21T01:45:36,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/5e3c64aae58f42e8b642ef631b4a6c78 is 50, key is test_row_0/B:col10/1732153534768/Put/seqid=0 2024-11-21T01:45:36,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742259_1435 (size=12301) 2024-11-21T01:45:36,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T01:45:36,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:36,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153596322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:36,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:36,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:36,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:36,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:36,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153596323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:36,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:36,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:36,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:36,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:36,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153596346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:36,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:36,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:36,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:36,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153596354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:36,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:36,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:36,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:36,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:36,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153596633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:36,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:36,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:36,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:36,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153596642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:36,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:36,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:36,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:36,659 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/5e3c64aae58f42e8b642ef631b4a6c78 2024-11-21T01:45:36,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/2cd8be6e175b4021892327fa22135729 is 50, key is test_row_0/C:col10/1732153534768/Put/seqid=0 2024-11-21T01:45:36,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153596664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:36,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:36,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:36,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:36,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742260_1436 (size=12301) 2024-11-21T01:45:36,682 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/2cd8be6e175b4021892327fa22135729 2024-11-21T01:45:36,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/10ee9b35fba0465c9e0cb7b8dd2dba0e as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/10ee9b35fba0465c9e0cb7b8dd2dba0e 2024-11-21T01:45:36,695 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/10ee9b35fba0465c9e0cb7b8dd2dba0e, entries=150, sequenceid=274, filesize=30.5 K 2024-11-21T01:45:36,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/5e3c64aae58f42e8b642ef631b4a6c78 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/5e3c64aae58f42e8b642ef631b4a6c78 2024-11-21T01:45:36,700 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/5e3c64aae58f42e8b642ef631b4a6c78, entries=150, sequenceid=274, filesize=12.0 K 2024-11-21T01:45:36,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:36,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153596682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:36,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:36,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:36,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:36,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/2cd8be6e175b4021892327fa22135729 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/2cd8be6e175b4021892327fa22135729 2024-11-21T01:45:36,729 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/2cd8be6e175b4021892327fa22135729, entries=150, sequenceid=274, filesize=12.0 K 2024-11-21T01:45:36,730 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 8b49a551695b0705f38348b99dc083b8 in 967ms, sequenceid=274, compaction requested=true 2024-11-21T01:45:36,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:36,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:36,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-21T01:45:36,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-21T01:45:36,736 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-21T01:45:36,736 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5980 sec 2024-11-21T01:45:36,738 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.6040 sec 2024-11-21T01:45:37,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b49a551695b0705f38348b99dc083b8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T01:45:37,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=A 2024-11-21T01:45:37,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:37,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=B 2024-11-21T01:45:37,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:37,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=C 2024-11-21T01:45:37,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:37,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:37,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e42cf4da458b4aeda4dc82f536de572f_8b49a551695b0705f38348b99dc083b8 is 50, key is test_row_0/A:col10/1732153537170/Put/seqid=0 2024-11-21T01:45:37,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742261_1437 (size=17534) 2024-11-21T01:45:37,203 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:45:37,207 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e42cf4da458b4aeda4dc82f536de572f_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e42cf4da458b4aeda4dc82f536de572f_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:37,207 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/363948b7a3be4c0094c4d05cea1df571, store: [table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:37,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/363948b7a3be4c0094c4d05cea1df571 is 175, key is test_row_0/A:col10/1732153537170/Put/seqid=0 2024-11-21T01:45:37,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742262_1438 (size=48639) 2024-11-21T01:45:37,227 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/363948b7a3be4c0094c4d05cea1df571 2024-11-21T01:45:37,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153597212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:37,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:37,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:37,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:37,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153597216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:37,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:37,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:37,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:37,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:37,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153597218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:37,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:37,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:37,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:37,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/9c79456349a14592afd315426d8d3250 is 50, key is test_row_0/B:col10/1732153537170/Put/seqid=0 2024-11-21T01:45:37,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742263_1439 (size=12301) 2024-11-21T01:45:37,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:37,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153597231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:37,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:37,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:37,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T01:45:37,283 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:45:37,283 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:45:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-21T01:45:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-21T01:45:37,285 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:45:37,286 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:45:37,286 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:45:37,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153597345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:37,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:37,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:37,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:37,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153597345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:37,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153597364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-21T01:45:37,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-21T01:45:37,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:37,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:37,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:37,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:37,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:37,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:37,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153597573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:37,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:37,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153597573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:37,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:37,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:37,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:37,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:37,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:37,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:37,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-21T01:45:37,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:37,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:37,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:37,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:37,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:37,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:37,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:37,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153597584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:37,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:37,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:37,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:37,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-21T01:45:37,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/9c79456349a14592afd315426d8d3250 2024-11-21T01:45:37,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/c998d5b6a2314e5996aded8b63dfcffb is 50, key is test_row_0/C:col10/1732153537170/Put/seqid=0 2024-11-21T01:45:37,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742264_1440 (size=12301) 2024-11-21T01:45:37,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-21T01:45:37,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:37,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:37,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:37,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:37,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:37,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:37,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:37,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153597884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:37,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:37,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:37,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:37,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153597884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:37,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:37,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:37,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:37,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:37,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-21T01:45:37,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:37,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:37,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:37,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:37,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:37,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:37,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-21T01:45:37,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:37,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153597903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:37,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:37,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:37,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:38,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-21T01:45:38,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:38,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:38,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:38,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:38,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:38,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:38,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/c998d5b6a2314e5996aded8b63dfcffb 2024-11-21T01:45:38,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/363948b7a3be4c0094c4d05cea1df571 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/363948b7a3be4c0094c4d05cea1df571 2024-11-21T01:45:38,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/363948b7a3be4c0094c4d05cea1df571, entries=250, sequenceid=294, filesize=47.5 K 2024-11-21T01:45:38,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/9c79456349a14592afd315426d8d3250 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/9c79456349a14592afd315426d8d3250 2024-11-21T01:45:38,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/9c79456349a14592afd315426d8d3250, entries=150, sequenceid=294, filesize=12.0 K 2024-11-21T01:45:38,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/c998d5b6a2314e5996aded8b63dfcffb as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/c998d5b6a2314e5996aded8b63dfcffb 2024-11-21T01:45:38,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/c998d5b6a2314e5996aded8b63dfcffb, entries=150, sequenceid=294, filesize=12.0 K 2024-11-21T01:45:38,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 8b49a551695b0705f38348b99dc083b8 in 936ms, sequenceid=294, compaction requested=true 2024-11-21T01:45:38,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:38,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b49a551695b0705f38348b99dc083b8:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:45:38,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:38,110 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:45:38,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b49a551695b0705f38348b99dc083b8:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:45:38,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:38,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b49a551695b0705f38348b99dc083b8:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:45:38,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:38,110 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:45:38,111 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151378 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:45:38,111 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 8b49a551695b0705f38348b99dc083b8/A is initiating minor compaction (all files) 2024-11-21T01:45:38,111 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b49a551695b0705f38348b99dc083b8/A in TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:38,111 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/d3ea01ecb3e34fe1a896bf860ddd60ed, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/654ffdd2377141b5aeb75658c9950098, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/10ee9b35fba0465c9e0cb7b8dd2dba0e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/363948b7a3be4c0094c4d05cea1df571] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp, totalSize=147.8 K 2024-11-21T01:45:38,111 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:38,111 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. files: [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/d3ea01ecb3e34fe1a896bf860ddd60ed, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/654ffdd2377141b5aeb75658c9950098, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/10ee9b35fba0465c9e0cb7b8dd2dba0e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/363948b7a3be4c0094c4d05cea1df571] 2024-11-21T01:45:38,112 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49534 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:45:38,112 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3ea01ecb3e34fe1a896bf860ddd60ed, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732153532325 2024-11-21T01:45:38,112 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 8b49a551695b0705f38348b99dc083b8/B is initiating minor compaction (all files) 2024-11-21T01:45:38,112 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b49a551695b0705f38348b99dc083b8/B in TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:38,112 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/1ae59e794c9c4de0abfa7be41bc7f332, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/e2d5f5dcb66440d091c22186ee287987, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/5e3c64aae58f42e8b642ef631b4a6c78, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/9c79456349a14592afd315426d8d3250] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp, totalSize=48.4 K 2024-11-21T01:45:38,112 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 654ffdd2377141b5aeb75658c9950098, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732153534535 2024-11-21T01:45:38,112 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10ee9b35fba0465c9e0cb7b8dd2dba0e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732153534762 2024-11-21T01:45:38,112 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ae59e794c9c4de0abfa7be41bc7f332, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732153532325 2024-11-21T01:45:38,113 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 363948b7a3be4c0094c4d05cea1df571, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732153535983 2024-11-21T01:45:38,113 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting e2d5f5dcb66440d091c22186ee287987, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732153534535 2024-11-21T01:45:38,113 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e3c64aae58f42e8b642ef631b4a6c78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732153534762 2024-11-21T01:45:38,113 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c79456349a14592afd315426d8d3250, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732153535983 2024-11-21T01:45:38,121 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:38,127 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112187a5d62620854c509f62eccf9e941fb7_8b49a551695b0705f38348b99dc083b8 store=[table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:38,129 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b49a551695b0705f38348b99dc083b8#B#compaction#373 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:38,130 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112187a5d62620854c509f62eccf9e941fb7_8b49a551695b0705f38348b99dc083b8, store=[table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:38,130 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112187a5d62620854c509f62eccf9e941fb7_8b49a551695b0705f38348b99dc083b8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:38,130 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/50218c27921640c7b459c6ae60ae250b is 50, key is test_row_0/B:col10/1732153537170/Put/seqid=0 2024-11-21T01:45:38,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742265_1441 (size=13017) 2024-11-21T01:45:38,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742266_1442 (size=4469) 2024-11-21T01:45:38,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-21T01:45:38,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:38,209 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 8b49a551695b0705f38348b99dc083b8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T01:45:38,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=A 2024-11-21T01:45:38,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:38,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=B 2024-11-21T01:45:38,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:38,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=C 2024-11-21T01:45:38,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:38,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212c911e590909434ba4feb45e37ac022e_8b49a551695b0705f38348b99dc083b8 is 50, key is test_row_0/A:col10/1732153537196/Put/seqid=0 2024-11-21T01:45:38,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742267_1443 (size=12454) 2024-11-21T01:45:38,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:45:38,240 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212c911e590909434ba4feb45e37ac022e_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212c911e590909434ba4feb45e37ac022e_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:38,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/4ad1aa8b85bc45ddb72b6debe9248265, store: [table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:38,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/4ad1aa8b85bc45ddb72b6debe9248265 is 175, key is test_row_0/A:col10/1732153537196/Put/seqid=0 2024-11-21T01:45:38,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:38,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. as already flushing 2024-11-21T01:45:38,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742268_1444 (size=31255) 2024-11-21T01:45:38,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-21T01:45:38,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:38,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153598421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:38,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:38,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:38,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:38,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:38,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57970 deadline: 1732153598423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:38,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:38,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153598424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:38,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:38,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:38,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:38,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:38,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153598426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:38,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:38,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:38,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:38,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:38,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:38,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:38,546 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b49a551695b0705f38348b99dc083b8#A#compaction#372 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:38,546 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/50218c27921640c7b459c6ae60ae250b as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/50218c27921640c7b459c6ae60ae250b 2024-11-21T01:45:38,547 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/4ae948c19b674bfeb68bd18edcc269a1 is 175, key is test_row_0/A:col10/1732153537170/Put/seqid=0 2024-11-21T01:45:38,555 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8b49a551695b0705f38348b99dc083b8/B of 8b49a551695b0705f38348b99dc083b8 into 50218c27921640c7b459c6ae60ae250b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:38,555 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:38,555 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., storeName=8b49a551695b0705f38348b99dc083b8/B, priority=12, startTime=1732153538109; duration=0sec 2024-11-21T01:45:38,555 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:38,555 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b49a551695b0705f38348b99dc083b8:B 2024-11-21T01:45:38,555 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:45:38,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742269_1445 (size=31971) 2024-11-21T01:45:38,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:38,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153598543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:38,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:38,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153598543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:38,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:38,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:38,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:38,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:38,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:38,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:38,562 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49534 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:45:38,562 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 8b49a551695b0705f38348b99dc083b8/C is initiating minor compaction (all files) 2024-11-21T01:45:38,562 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b49a551695b0705f38348b99dc083b8/C in TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:38,562 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/62a0aeaee8f0448ab551aadcc785df63, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/14db3c0008514778a94e5e71545b7921, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/2cd8be6e175b4021892327fa22135729, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/c998d5b6a2314e5996aded8b63dfcffb] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp, totalSize=48.4 K 2024-11-21T01:45:38,563 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 62a0aeaee8f0448ab551aadcc785df63, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732153532325 2024-11-21T01:45:38,563 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 14db3c0008514778a94e5e71545b7921, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732153534535 2024-11-21T01:45:38,563 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cd8be6e175b4021892327fa22135729, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732153534762 2024-11-21T01:45:38,564 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting c998d5b6a2314e5996aded8b63dfcffb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732153535983 2024-11-21T01:45:38,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:38,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153598553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:38,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:38,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:38,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:38,573 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/4ae948c19b674bfeb68bd18edcc269a1 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/4ae948c19b674bfeb68bd18edcc269a1 2024-11-21T01:45:38,578 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8b49a551695b0705f38348b99dc083b8/A of 8b49a551695b0705f38348b99dc083b8 into 4ae948c19b674bfeb68bd18edcc269a1(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:38,578 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:38,578 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., storeName=8b49a551695b0705f38348b99dc083b8/A, priority=12, startTime=1732153538109; duration=0sec 2024-11-21T01:45:38,578 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:38,578 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b49a551695b0705f38348b99dc083b8:A 2024-11-21T01:45:38,582 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b49a551695b0705f38348b99dc083b8#C#compaction#375 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:38,582 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/31d3ef3626ef44f19820ce1a04efddf1 is 50, key is test_row_0/C:col10/1732153537170/Put/seqid=0 2024-11-21T01:45:38,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742270_1446 (size=13017) 2024-11-21T01:45:38,667 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=306, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/4ad1aa8b85bc45ddb72b6debe9248265 2024-11-21T01:45:38,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/23ccfa03a2d840e3b20d0121853c7968 is 50, key is test_row_0/B:col10/1732153537196/Put/seqid=0 2024-11-21T01:45:38,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742271_1447 (size=12301) 2024-11-21T01:45:38,683 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/23ccfa03a2d840e3b20d0121853c7968 2024-11-21T01:45:38,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/1f4439a4d1304900900e5e3cfd040173 is 50, key is test_row_0/C:col10/1732153537196/Put/seqid=0 2024-11-21T01:45:38,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742272_1448 (size=12301) 2024-11-21T01:45:38,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153598762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:38,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:38,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:38,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153598763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:38,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:38,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:38,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:38,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:38,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:38,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153598774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:38,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:38,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:38,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:39,012 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/31d3ef3626ef44f19820ce1a04efddf1 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/31d3ef3626ef44f19820ce1a04efddf1 2024-11-21T01:45:39,017 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8b49a551695b0705f38348b99dc083b8/C of 8b49a551695b0705f38348b99dc083b8 into 31d3ef3626ef44f19820ce1a04efddf1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:39,017 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:39,017 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., storeName=8b49a551695b0705f38348b99dc083b8/C, priority=12, startTime=1732153538110; duration=0sec 2024-11-21T01:45:39,018 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:39,018 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b49a551695b0705f38348b99dc083b8:C 2024-11-21T01:45:39,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:39,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58054 deadline: 1732153599083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:39,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:39,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:39,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:39,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:39,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58028 deadline: 1732153599086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:39,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:39,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:39,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:39,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:39,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58006 deadline: 1732153599093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:39,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:39,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:39,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:45:39,111 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/1f4439a4d1304900900e5e3cfd040173 2024-11-21T01:45:39,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/4ad1aa8b85bc45ddb72b6debe9248265 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/4ad1aa8b85bc45ddb72b6debe9248265 2024-11-21T01:45:39,139 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/4ad1aa8b85bc45ddb72b6debe9248265, entries=150, sequenceid=306, filesize=30.5 K 2024-11-21T01:45:39,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/23ccfa03a2d840e3b20d0121853c7968 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/23ccfa03a2d840e3b20d0121853c7968 2024-11-21T01:45:39,145 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/23ccfa03a2d840e3b20d0121853c7968, entries=150, sequenceid=306, filesize=12.0 K 2024-11-21T01:45:39,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/1f4439a4d1304900900e5e3cfd040173 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/1f4439a4d1304900900e5e3cfd040173 2024-11-21T01:45:39,150 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/1f4439a4d1304900900e5e3cfd040173, entries=150, sequenceid=306, filesize=12.0 K 2024-11-21T01:45:39,153 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8b49a551695b0705f38348b99dc083b8 in 944ms, sequenceid=306, compaction requested=false 2024-11-21T01:45:39,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:39,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:39,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-21T01:45:39,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-21T01:45:39,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-21T01:45:39,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8690 sec 2024-11-21T01:45:39,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.8750 sec 2024-11-21T01:45:39,336 INFO [Thread-1625 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1625. 2024-11-21T01:45:39,336 DEBUG [Thread-1625 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:45:39,336 DEBUG [Thread-1625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:39,336 INFO [Thread-1629 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1629. 2024-11-21T01:45:39,336 DEBUG [Thread-1629 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:45:39,336 DEBUG [Thread-1625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:39,336 DEBUG [Thread-1629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:39,336 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:45:39,337 DEBUG [Thread-1629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:39,337 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:45:39,337 INFO [Thread-1627 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1627. 2024-11-21T01:45:39,337 DEBUG [Thread-1627 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:45:39,337 DEBUG [Thread-1627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:39,337 DEBUG [Thread-1627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:39,337 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:45:39,344 INFO [Thread-1621 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1621. 2024-11-21T01:45:39,344 DEBUG [Thread-1621 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:45:39,344 DEBUG [Thread-1621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:39,344 INFO [Thread-1623 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1623. 2024-11-21T01:45:39,344 DEBUG [Thread-1623 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:45:39,344 DEBUG [Thread-1621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:39,344 DEBUG [Thread-1623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:39,344 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:45:39,344 DEBUG [Thread-1623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:39,344 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:45:39,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-21T01:45:39,433 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:45:39,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:39,454 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b49a551695b0705f38348b99dc083b8 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-21T01:45:39,454 INFO [Thread-1612 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1612. 2024-11-21T01:45:39,454 DEBUG [Thread-1612 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:45:39,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=A 2024-11-21T01:45:39,454 DEBUG [Thread-1612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:39,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:45:39,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=B 2024-11-21T01:45:39,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:45:39,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=C 2024-11-21T01:45:39,454 DEBUG [Thread-1612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:39,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:45:39,455 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:45:39,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411218b437536332e4692900f45323c3f2825_8b49a551695b0705f38348b99dc083b8 is 50, key is test_row_0/A:col10/1732153539453/Put/seqid=0 2024-11-21T01:45:39,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742273_1449 (size=12454) 2024-11-21T01:45:39,594 INFO [Thread-1614 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1614. 2024-11-21T01:45:39,594 DEBUG [Thread-1614 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:45:39,594 DEBUG [Thread-1614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:39,594 DEBUG [Thread-1614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:39,594 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:45:39,604 INFO [Thread-1616 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1616. 2024-11-21T01:45:39,604 DEBUG [Thread-1616 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:45:39,604 DEBUG [Thread-1616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:39,604 DEBUG [Thread-1616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:39,604 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:45:39,613 INFO [Thread-1618 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1618. 2024-11-21T01:45:39,614 DEBUG [Thread-1618 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:45:39,614 DEBUG [Thread-1618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:39,614 DEBUG [Thread-1618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:39,614 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:45:39,865 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:45:39,869 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411218b437536332e4692900f45323c3f2825_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411218b437536332e4692900f45323c3f2825_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:39,870 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/60da65e168ed412eaa7e5a6d8ec61ff3, store: [table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:39,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/60da65e168ed412eaa7e5a6d8ec61ff3 is 175, key is test_row_0/A:col10/1732153539453/Put/seqid=0 2024-11-21T01:45:39,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742274_1450 (size=31255) 2024-11-21T01:45:40,274 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=330, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/60da65e168ed412eaa7e5a6d8ec61ff3 2024-11-21T01:45:40,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/5410fe34e54e46a39d2ad7ecdb2d5b75 is 50, key is test_row_0/B:col10/1732153539453/Put/seqid=0 2024-11-21T01:45:40,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742275_1451 (size=12301) 2024-11-21T01:45:40,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/5410fe34e54e46a39d2ad7ecdb2d5b75 2024-11-21T01:45:40,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/25af158dcffd40f4a6814fc4a9702304 is 50, key is test_row_0/C:col10/1732153539453/Put/seqid=0 2024-11-21T01:45:40,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742276_1452 (size=12301) 2024-11-21T01:45:41,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/25af158dcffd40f4a6814fc4a9702304 2024-11-21T01:45:41,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/60da65e168ed412eaa7e5a6d8ec61ff3 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/60da65e168ed412eaa7e5a6d8ec61ff3 2024-11-21T01:45:41,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/60da65e168ed412eaa7e5a6d8ec61ff3, entries=150, sequenceid=330, filesize=30.5 K 2024-11-21T01:45:41,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/5410fe34e54e46a39d2ad7ecdb2d5b75 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/5410fe34e54e46a39d2ad7ecdb2d5b75 2024-11-21T01:45:41,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/5410fe34e54e46a39d2ad7ecdb2d5b75, entries=150, sequenceid=330, filesize=12.0 K 2024-11-21T01:45:41,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/25af158dcffd40f4a6814fc4a9702304 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/25af158dcffd40f4a6814fc4a9702304 2024-11-21T01:45:41,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/25af158dcffd40f4a6814fc4a9702304, entries=150, sequenceid=330, filesize=12.0 K 2024-11-21T01:45:41,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=20.13 KB/20610 for 8b49a551695b0705f38348b99dc083b8 in 1665ms, sequenceid=330, compaction requested=true 2024-11-21T01:45:41,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:41,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b49a551695b0705f38348b99dc083b8:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:45:41,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:41,119 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:41,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b49a551695b0705f38348b99dc083b8:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:45:41,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:41,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b49a551695b0705f38348b99dc083b8:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:45:41,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:41,119 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:41,120 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:41,120 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 8b49a551695b0705f38348b99dc083b8/A is initiating minor compaction (all files) 2024-11-21T01:45:41,120 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:41,120 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b49a551695b0705f38348b99dc083b8/A in TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:41,120 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 8b49a551695b0705f38348b99dc083b8/B is initiating minor compaction (all files) 2024-11-21T01:45:41,120 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b49a551695b0705f38348b99dc083b8/B in TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:41,120 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/4ae948c19b674bfeb68bd18edcc269a1, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/4ad1aa8b85bc45ddb72b6debe9248265, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/60da65e168ed412eaa7e5a6d8ec61ff3] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp, totalSize=92.3 K 2024-11-21T01:45:41,120 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:41,120 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/50218c27921640c7b459c6ae60ae250b, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/23ccfa03a2d840e3b20d0121853c7968, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/5410fe34e54e46a39d2ad7ecdb2d5b75] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp, totalSize=36.7 K 2024-11-21T01:45:41,120 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. files: [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/4ae948c19b674bfeb68bd18edcc269a1, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/4ad1aa8b85bc45ddb72b6debe9248265, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/60da65e168ed412eaa7e5a6d8ec61ff3] 2024-11-21T01:45:41,121 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ae948c19b674bfeb68bd18edcc269a1, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732153535983 2024-11-21T01:45:41,121 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 50218c27921640c7b459c6ae60ae250b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732153535983 2024-11-21T01:45:41,121 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ad1aa8b85bc45ddb72b6debe9248265, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732153537196 2024-11-21T01:45:41,121 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 23ccfa03a2d840e3b20d0121853c7968, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732153537196 2024-11-21T01:45:41,121 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60da65e168ed412eaa7e5a6d8ec61ff3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732153538419 2024-11-21T01:45:41,121 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 5410fe34e54e46a39d2ad7ecdb2d5b75, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732153538419 2024-11-21T01:45:41,128 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b49a551695b0705f38348b99dc083b8#B#compaction#381 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:41,128 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/6446a125734c46a49c0b65cf41f39f76 is 50, key is test_row_0/B:col10/1732153539453/Put/seqid=0 2024-11-21T01:45:41,131 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:41,134 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121bffc851b18314c5e8be2c28a44f33e16_8b49a551695b0705f38348b99dc083b8 store=[table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:41,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742277_1453 (size=13119) 2024-11-21T01:45:41,149 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121bffc851b18314c5e8be2c28a44f33e16_8b49a551695b0705f38348b99dc083b8, store=[table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:41,150 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121bffc851b18314c5e8be2c28a44f33e16_8b49a551695b0705f38348b99dc083b8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:41,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742278_1454 (size=4469) 2024-11-21T01:45:41,540 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/6446a125734c46a49c0b65cf41f39f76 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/6446a125734c46a49c0b65cf41f39f76 2024-11-21T01:45:41,544 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b49a551695b0705f38348b99dc083b8/B of 8b49a551695b0705f38348b99dc083b8 into 6446a125734c46a49c0b65cf41f39f76(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:41,544 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:41,544 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., storeName=8b49a551695b0705f38348b99dc083b8/B, priority=13, startTime=1732153541119; duration=0sec 2024-11-21T01:45:41,544 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:41,544 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b49a551695b0705f38348b99dc083b8:B 2024-11-21T01:45:41,544 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:41,545 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:41,545 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 8b49a551695b0705f38348b99dc083b8/C is initiating minor compaction (all files) 2024-11-21T01:45:41,545 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b49a551695b0705f38348b99dc083b8/C in TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:41,545 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/31d3ef3626ef44f19820ce1a04efddf1, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/1f4439a4d1304900900e5e3cfd040173, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/25af158dcffd40f4a6814fc4a9702304] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp, totalSize=36.7 K 2024-11-21T01:45:41,546 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 31d3ef3626ef44f19820ce1a04efddf1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732153535983 2024-11-21T01:45:41,546 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f4439a4d1304900900e5e3cfd040173, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732153537196 2024-11-21T01:45:41,546 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 25af158dcffd40f4a6814fc4a9702304, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732153538419 2024-11-21T01:45:41,554 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b49a551695b0705f38348b99dc083b8#C#compaction#383 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:41,555 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/cb407feb1af2490194752531d359eb79 is 50, key is test_row_0/C:col10/1732153539453/Put/seqid=0 2024-11-21T01:45:41,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742279_1455 (size=13119) 2024-11-21T01:45:41,560 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b49a551695b0705f38348b99dc083b8#A#compaction#382 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:41,561 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/09399aa708104c2d9bedbdf647be6cbe is 175, key is test_row_0/A:col10/1732153539453/Put/seqid=0 2024-11-21T01:45:41,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742280_1456 (size=32073) 2024-11-21T01:45:41,572 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/cb407feb1af2490194752531d359eb79 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/cb407feb1af2490194752531d359eb79 2024-11-21T01:45:41,577 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b49a551695b0705f38348b99dc083b8/C of 8b49a551695b0705f38348b99dc083b8 into cb407feb1af2490194752531d359eb79(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:41,577 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:41,577 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., storeName=8b49a551695b0705f38348b99dc083b8/C, priority=13, startTime=1732153541119; duration=0sec 2024-11-21T01:45:41,577 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:41,577 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b49a551695b0705f38348b99dc083b8:C 2024-11-21T01:45:41,970 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/09399aa708104c2d9bedbdf647be6cbe as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/09399aa708104c2d9bedbdf647be6cbe 2024-11-21T01:45:41,974 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b49a551695b0705f38348b99dc083b8/A of 8b49a551695b0705f38348b99dc083b8 into 09399aa708104c2d9bedbdf647be6cbe(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:41,974 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b49a551695b0705f38348b99dc083b8: 2024-11-21T01:45:41,974 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., storeName=8b49a551695b0705f38348b99dc083b8/A, priority=13, startTime=1732153541119; duration=0sec 2024-11-21T01:45:41,974 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:41,974 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b49a551695b0705f38348b99dc083b8:A 2024-11-21T01:45:43,244 INFO [Thread-1610 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1610. 2024-11-21T01:45:43,244 DEBUG [Thread-1610 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:45:43,244 DEBUG [Thread-1610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:43,244 DEBUG [Thread-1610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:43,244 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:45:43,245 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T01:45:43,245 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-21T01:45:43,245 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-21T01:45:43,245 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-21T01:45:43,245 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-21T01:45:43,245 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-21T01:45:43,245 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T01:45:43,245 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T01:45:43,245 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 902 2024-11-21T01:45:43,245 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2700 rows 2024-11-21T01:45:43,245 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 901 2024-11-21T01:45:43,245 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2701 rows 2024-11-21T01:45:43,245 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 902 2024-11-21T01:45:43,245 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2704 rows 2024-11-21T01:45:43,245 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 900 2024-11-21T01:45:43,245 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2695 rows 2024-11-21T01:45:43,245 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 902 2024-11-21T01:45:43,245 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2705 rows 2024-11-21T01:45:43,245 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T01:45:43,245 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T01:45:43,245 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:43,245 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:43,245 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:45:43,246 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-21T01:45:43,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T01:45:43,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T01:45:43,249 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732153543249"}]},"ts":"1732153543249"} 2024-11-21T01:45:43,251 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T01:45:43,251 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T01:45:43,251 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T01:45:43,252 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b49a551695b0705f38348b99dc083b8, UNASSIGN}] 2024-11-21T01:45:43,253 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b49a551695b0705f38348b99dc083b8, UNASSIGN 2024-11-21T01:45:43,254 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=8b49a551695b0705f38348b99dc083b8, regionState=CLOSING, regionLocation=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:43,255 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b49a551695b0705f38348b99dc083b8, UNASSIGN because future has completed 2024-11-21T01:45:43,255 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T01:45:43,255 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359}] 2024-11-21T01:45:43,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T01:45:43,407 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] handler.UnassignRegionHandler(122): Close 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:43,407 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T01:45:43,408 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1722): Closing 8b49a551695b0705f38348b99dc083b8, disabling compactions & flushes 2024-11-21T01:45:43,408 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:43,408 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:43,408 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. after waiting 0 ms 2024-11-21T01:45:43,408 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:43,408 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(2902): Flushing 8b49a551695b0705f38348b99dc083b8 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-21T01:45:43,408 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=A 2024-11-21T01:45:43,408 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:43,408 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=B 2024-11-21T01:45:43,408 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:43,408 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b49a551695b0705f38348b99dc083b8, store=C 2024-11-21T01:45:43,408 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:43,415 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210ab68cf9c51041c3a4461bda83584646_8b49a551695b0705f38348b99dc083b8 is 50, key is test_row_0/A:col10/1732153539612/Put/seqid=0 2024-11-21T01:45:43,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742281_1457 (size=9914) 2024-11-21T01:45:43,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T01:45:43,819 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:45:43,822 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210ab68cf9c51041c3a4461bda83584646_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210ab68cf9c51041c3a4461bda83584646_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:43,823 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/0d14237613ab40e28cf8852bf9945e2a, store: [table=TestAcidGuarantees family=A region=8b49a551695b0705f38348b99dc083b8] 2024-11-21T01:45:43,824 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/0d14237613ab40e28cf8852bf9945e2a is 175, key is test_row_0/A:col10/1732153539612/Put/seqid=0 2024-11-21T01:45:43,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742282_1458 (size=22561) 2024-11-21T01:45:43,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T01:45:44,228 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=340, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/0d14237613ab40e28cf8852bf9945e2a 2024-11-21T01:45:44,234 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/518badf0b52445868d601c69d8a7bd8e is 50, key is test_row_0/B:col10/1732153539612/Put/seqid=0 2024-11-21T01:45:44,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742283_1459 (size=9857) 2024-11-21T01:45:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T01:45:44,637 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/518badf0b52445868d601c69d8a7bd8e 2024-11-21T01:45:44,645 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/a102a4e7b08b4916a6d8e1f3f46d3d26 is 50, key is test_row_0/C:col10/1732153539612/Put/seqid=0 2024-11-21T01:45:44,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742284_1460 (size=9857) 2024-11-21T01:45:45,049 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/a102a4e7b08b4916a6d8e1f3f46d3d26 2024-11-21T01:45:45,053 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/A/0d14237613ab40e28cf8852bf9945e2a as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/0d14237613ab40e28cf8852bf9945e2a 2024-11-21T01:45:45,057 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/0d14237613ab40e28cf8852bf9945e2a, entries=100, sequenceid=340, filesize=22.0 K 2024-11-21T01:45:45,058 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/B/518badf0b52445868d601c69d8a7bd8e as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/518badf0b52445868d601c69d8a7bd8e 2024-11-21T01:45:45,062 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/518badf0b52445868d601c69d8a7bd8e, entries=100, sequenceid=340, filesize=9.6 K 2024-11-21T01:45:45,062 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/.tmp/C/a102a4e7b08b4916a6d8e1f3f46d3d26 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/a102a4e7b08b4916a6d8e1f3f46d3d26 2024-11-21T01:45:45,065 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/a102a4e7b08b4916a6d8e1f3f46d3d26, entries=100, sequenceid=340, filesize=9.6 K 2024-11-21T01:45:45,066 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 8b49a551695b0705f38348b99dc083b8 in 1658ms, sequenceid=340, compaction requested=false 2024-11-21T01:45:45,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/56f130e31c154eff977caf50c150fb57, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/cdb9f21452794da19c415df8c8fcdb31, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/3244f0623d3643a89ab93c52e4f1c546, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/3401b2ed2ff44b9faed5f9421e9049dd, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/28cfdc16f0574784a69bdad8e5a367f0, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/28aebc80325941b58257c041c21f3985, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/2d287b3d3d6641c28d6540fde96aea48, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/b203f497dc80421693bcfe0a50e47d27, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/582f75afa64a4c60970f5e09219b70f6, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/2186f7aba1c747529fa03da7c41466ab, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/9171802b7302428eb6ac2f647c18121b, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/ff292309d21c4d699aaff07dacf4ec10, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/874db9e4775244fc982a874eb51fed47, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/95abd6145e1d478da742a548bcb39b1b, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/6c796a97dd594cbeb33371476fe7b29c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/f18d43b3ff1f44cf8931e8990f8e3242, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/95dbda7768ee4df689309f1470b51e90, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/d3ea01ecb3e34fe1a896bf860ddd60ed, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/654ffdd2377141b5aeb75658c9950098, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/10ee9b35fba0465c9e0cb7b8dd2dba0e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/363948b7a3be4c0094c4d05cea1df571, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/4ae948c19b674bfeb68bd18edcc269a1, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/4ad1aa8b85bc45ddb72b6debe9248265, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/60da65e168ed412eaa7e5a6d8ec61ff3] to archive 2024-11-21T01:45:45,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T01:45:45,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/56f130e31c154eff977caf50c150fb57 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/56f130e31c154eff977caf50c150fb57 2024-11-21T01:45:45,070 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/cdb9f21452794da19c415df8c8fcdb31 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/cdb9f21452794da19c415df8c8fcdb31 2024-11-21T01:45:45,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/3244f0623d3643a89ab93c52e4f1c546 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/3244f0623d3643a89ab93c52e4f1c546 2024-11-21T01:45:45,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/3401b2ed2ff44b9faed5f9421e9049dd to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/3401b2ed2ff44b9faed5f9421e9049dd 2024-11-21T01:45:45,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/28cfdc16f0574784a69bdad8e5a367f0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/28cfdc16f0574784a69bdad8e5a367f0 2024-11-21T01:45:45,073 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/28aebc80325941b58257c041c21f3985 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/28aebc80325941b58257c041c21f3985 2024-11-21T01:45:45,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/2d287b3d3d6641c28d6540fde96aea48 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/2d287b3d3d6641c28d6540fde96aea48 2024-11-21T01:45:45,075 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/b203f497dc80421693bcfe0a50e47d27 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/b203f497dc80421693bcfe0a50e47d27 2024-11-21T01:45:45,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/582f75afa64a4c60970f5e09219b70f6 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/582f75afa64a4c60970f5e09219b70f6 2024-11-21T01:45:45,077 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/2186f7aba1c747529fa03da7c41466ab to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/2186f7aba1c747529fa03da7c41466ab 2024-11-21T01:45:45,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/9171802b7302428eb6ac2f647c18121b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/9171802b7302428eb6ac2f647c18121b 2024-11-21T01:45:45,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/ff292309d21c4d699aaff07dacf4ec10 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/ff292309d21c4d699aaff07dacf4ec10 2024-11-21T01:45:45,080 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/874db9e4775244fc982a874eb51fed47 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/874db9e4775244fc982a874eb51fed47 2024-11-21T01:45:45,080 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/95abd6145e1d478da742a548bcb39b1b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/95abd6145e1d478da742a548bcb39b1b 2024-11-21T01:45:45,081 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/6c796a97dd594cbeb33371476fe7b29c to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/6c796a97dd594cbeb33371476fe7b29c 2024-11-21T01:45:45,082 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/f18d43b3ff1f44cf8931e8990f8e3242 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/f18d43b3ff1f44cf8931e8990f8e3242 2024-11-21T01:45:45,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/95dbda7768ee4df689309f1470b51e90 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/95dbda7768ee4df689309f1470b51e90 2024-11-21T01:45:45,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/d3ea01ecb3e34fe1a896bf860ddd60ed to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/d3ea01ecb3e34fe1a896bf860ddd60ed 2024-11-21T01:45:45,084 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/654ffdd2377141b5aeb75658c9950098 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/654ffdd2377141b5aeb75658c9950098 2024-11-21T01:45:45,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/10ee9b35fba0465c9e0cb7b8dd2dba0e to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/10ee9b35fba0465c9e0cb7b8dd2dba0e 2024-11-21T01:45:45,086 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/363948b7a3be4c0094c4d05cea1df571 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/363948b7a3be4c0094c4d05cea1df571 2024-11-21T01:45:45,087 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/4ae948c19b674bfeb68bd18edcc269a1 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/4ae948c19b674bfeb68bd18edcc269a1 2024-11-21T01:45:45,088 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/4ad1aa8b85bc45ddb72b6debe9248265 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/4ad1aa8b85bc45ddb72b6debe9248265 2024-11-21T01:45:45,089 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/60da65e168ed412eaa7e5a6d8ec61ff3 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/60da65e168ed412eaa7e5a6d8ec61ff3 2024-11-21T01:45:45,090 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/19dae5c8400b4dc085ce873ba4b748ee, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/2b822ffaa486456fa3b6bdeb15fc0ed1, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/57b49760443d4e198bee85b45b73b486, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/2ba45ebb61f6466cb22baa7bfdd3836f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/8c668b396aa64f22b0b67ed74f818d5e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/e8968163e02b47d4baf819c17366c668, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/048bb9d5ca74410a85fd4b66e506b69c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/3e1897a7f5c34bf6a2217c01df566541, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/a23b19f7b229494d8a13f866e6100406, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/283352b73604400198ca12e6e84db867, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/82ad106da1df4d7d891706ac67f6fbcf, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/8c3744a1f9824def94abfd75c26d0363, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/bcd2e179c0a04fc194e938d38a8c6fb6, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/a00034a26e884793a27a5e6fec667e1a, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/bc8c2aa527814cb1aefad5bda4566648, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/19bfef38f4c24487b2d936c29a4bce9f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/1ae59e794c9c4de0abfa7be41bc7f332, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/28c6ee3971d44173ae3b2af4533a6144, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/e2d5f5dcb66440d091c22186ee287987, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/5e3c64aae58f42e8b642ef631b4a6c78, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/50218c27921640c7b459c6ae60ae250b, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/9c79456349a14592afd315426d8d3250, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/23ccfa03a2d840e3b20d0121853c7968, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/5410fe34e54e46a39d2ad7ecdb2d5b75] to archive 2024-11-21T01:45:45,091 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T01:45:45,092 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/19dae5c8400b4dc085ce873ba4b748ee to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/19dae5c8400b4dc085ce873ba4b748ee 2024-11-21T01:45:45,093 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/2b822ffaa486456fa3b6bdeb15fc0ed1 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/2b822ffaa486456fa3b6bdeb15fc0ed1 2024-11-21T01:45:45,094 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/57b49760443d4e198bee85b45b73b486 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/57b49760443d4e198bee85b45b73b486 2024-11-21T01:45:45,094 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/2ba45ebb61f6466cb22baa7bfdd3836f to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/2ba45ebb61f6466cb22baa7bfdd3836f 2024-11-21T01:45:45,095 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/8c668b396aa64f22b0b67ed74f818d5e to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/8c668b396aa64f22b0b67ed74f818d5e 2024-11-21T01:45:45,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/e8968163e02b47d4baf819c17366c668 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/e8968163e02b47d4baf819c17366c668 2024-11-21T01:45:45,097 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/048bb9d5ca74410a85fd4b66e506b69c to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/048bb9d5ca74410a85fd4b66e506b69c 2024-11-21T01:45:45,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/3e1897a7f5c34bf6a2217c01df566541 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/3e1897a7f5c34bf6a2217c01df566541 2024-11-21T01:45:45,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/a23b19f7b229494d8a13f866e6100406 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/a23b19f7b229494d8a13f866e6100406 2024-11-21T01:45:45,100 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/283352b73604400198ca12e6e84db867 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/283352b73604400198ca12e6e84db867 2024-11-21T01:45:45,101 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/82ad106da1df4d7d891706ac67f6fbcf to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/82ad106da1df4d7d891706ac67f6fbcf 2024-11-21T01:45:45,102 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/8c3744a1f9824def94abfd75c26d0363 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/8c3744a1f9824def94abfd75c26d0363 2024-11-21T01:45:45,103 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/bcd2e179c0a04fc194e938d38a8c6fb6 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/bcd2e179c0a04fc194e938d38a8c6fb6 2024-11-21T01:45:45,104 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/a00034a26e884793a27a5e6fec667e1a to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/a00034a26e884793a27a5e6fec667e1a 2024-11-21T01:45:45,104 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/bc8c2aa527814cb1aefad5bda4566648 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/bc8c2aa527814cb1aefad5bda4566648 2024-11-21T01:45:45,105 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/19bfef38f4c24487b2d936c29a4bce9f to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/19bfef38f4c24487b2d936c29a4bce9f 2024-11-21T01:45:45,106 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/1ae59e794c9c4de0abfa7be41bc7f332 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/1ae59e794c9c4de0abfa7be41bc7f332 2024-11-21T01:45:45,107 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/28c6ee3971d44173ae3b2af4533a6144 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/28c6ee3971d44173ae3b2af4533a6144 2024-11-21T01:45:45,108 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/e2d5f5dcb66440d091c22186ee287987 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/e2d5f5dcb66440d091c22186ee287987 2024-11-21T01:45:45,109 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/5e3c64aae58f42e8b642ef631b4a6c78 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/5e3c64aae58f42e8b642ef631b4a6c78 2024-11-21T01:45:45,110 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/50218c27921640c7b459c6ae60ae250b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/50218c27921640c7b459c6ae60ae250b 2024-11-21T01:45:45,111 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/9c79456349a14592afd315426d8d3250 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/9c79456349a14592afd315426d8d3250 2024-11-21T01:45:45,111 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/23ccfa03a2d840e3b20d0121853c7968 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/23ccfa03a2d840e3b20d0121853c7968 2024-11-21T01:45:45,112 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/5410fe34e54e46a39d2ad7ecdb2d5b75 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/5410fe34e54e46a39d2ad7ecdb2d5b75 2024-11-21T01:45:45,113 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/1b22e72aa13e451bbf7d12c3efd81ee9, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/3f0ac36298e14fa498db4bf2a7789abd, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/ef8dff03592947a895f97d24421cb32d, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/6b8ec015a88344f2a6b3a85df6384360, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/d407b42ca0ae462e9da3a2e3991e847e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/67749dbdfe8c4e129cc2dc678452a998, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/72fe32ac086f4028a3aaf8ebd4733bd3, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/9843668da1c54b9b81abbe76f65b4d72, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/afc33746fb5c416d8de036ae815bb438, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/6b8cab8b2c3c48039bad60afbb36569a, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/e0f5fde9b98547eeb5ad2968dcb6a2af, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/6f66b42c28f14fd4ad19701581135f34, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/2e66ae67eb6541aa9a3a2d40ec6f92e6, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/8cd012d06bda4884991ecb1fcf846226, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/51104c3b09f94e58994467b6e27e73d2, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/dc464de248fc41f59548ddeeb886892d, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/62a0aeaee8f0448ab551aadcc785df63, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/28832761adf54e1babe97b32cdac4167, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/14db3c0008514778a94e5e71545b7921, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/2cd8be6e175b4021892327fa22135729, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/31d3ef3626ef44f19820ce1a04efddf1, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/c998d5b6a2314e5996aded8b63dfcffb, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/1f4439a4d1304900900e5e3cfd040173, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/25af158dcffd40f4a6814fc4a9702304] to archive 2024-11-21T01:45:45,114 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T01:45:45,115 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/1b22e72aa13e451bbf7d12c3efd81ee9 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/1b22e72aa13e451bbf7d12c3efd81ee9 2024-11-21T01:45:45,116 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/3f0ac36298e14fa498db4bf2a7789abd to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/3f0ac36298e14fa498db4bf2a7789abd 2024-11-21T01:45:45,117 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/ef8dff03592947a895f97d24421cb32d to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/ef8dff03592947a895f97d24421cb32d 2024-11-21T01:45:45,117 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/6b8ec015a88344f2a6b3a85df6384360 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/6b8ec015a88344f2a6b3a85df6384360 2024-11-21T01:45:45,118 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/d407b42ca0ae462e9da3a2e3991e847e to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/d407b42ca0ae462e9da3a2e3991e847e 2024-11-21T01:45:45,119 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/67749dbdfe8c4e129cc2dc678452a998 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/67749dbdfe8c4e129cc2dc678452a998 2024-11-21T01:45:45,120 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/72fe32ac086f4028a3aaf8ebd4733bd3 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/72fe32ac086f4028a3aaf8ebd4733bd3 2024-11-21T01:45:45,120 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/9843668da1c54b9b81abbe76f65b4d72 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/9843668da1c54b9b81abbe76f65b4d72 2024-11-21T01:45:45,121 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/afc33746fb5c416d8de036ae815bb438 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/afc33746fb5c416d8de036ae815bb438 2024-11-21T01:45:45,122 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/6b8cab8b2c3c48039bad60afbb36569a to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/6b8cab8b2c3c48039bad60afbb36569a 2024-11-21T01:45:45,123 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/e0f5fde9b98547eeb5ad2968dcb6a2af to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/e0f5fde9b98547eeb5ad2968dcb6a2af 2024-11-21T01:45:45,123 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/6f66b42c28f14fd4ad19701581135f34 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/6f66b42c28f14fd4ad19701581135f34 2024-11-21T01:45:45,124 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/2e66ae67eb6541aa9a3a2d40ec6f92e6 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/2e66ae67eb6541aa9a3a2d40ec6f92e6 2024-11-21T01:45:45,125 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/8cd012d06bda4884991ecb1fcf846226 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/8cd012d06bda4884991ecb1fcf846226 2024-11-21T01:45:45,126 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/51104c3b09f94e58994467b6e27e73d2 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/51104c3b09f94e58994467b6e27e73d2 2024-11-21T01:45:45,127 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/dc464de248fc41f59548ddeeb886892d to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/dc464de248fc41f59548ddeeb886892d 2024-11-21T01:45:45,127 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/62a0aeaee8f0448ab551aadcc785df63 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/62a0aeaee8f0448ab551aadcc785df63 2024-11-21T01:45:45,128 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/28832761adf54e1babe97b32cdac4167 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/28832761adf54e1babe97b32cdac4167 2024-11-21T01:45:45,129 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/14db3c0008514778a94e5e71545b7921 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/14db3c0008514778a94e5e71545b7921 2024-11-21T01:45:45,130 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/2cd8be6e175b4021892327fa22135729 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/2cd8be6e175b4021892327fa22135729 2024-11-21T01:45:45,130 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/31d3ef3626ef44f19820ce1a04efddf1 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/31d3ef3626ef44f19820ce1a04efddf1 2024-11-21T01:45:45,131 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/c998d5b6a2314e5996aded8b63dfcffb to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/c998d5b6a2314e5996aded8b63dfcffb 2024-11-21T01:45:45,132 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/1f4439a4d1304900900e5e3cfd040173 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/1f4439a4d1304900900e5e3cfd040173 2024-11-21T01:45:45,133 DEBUG [StoreCloser-TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/25af158dcffd40f4a6814fc4a9702304 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/25af158dcffd40f4a6814fc4a9702304 2024-11-21T01:45:45,136 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/recovered.edits/343.seqid, newMaxSeqId=343, maxSeqId=4 2024-11-21T01:45:45,136 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. 2024-11-21T01:45:45,137 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1676): Region close journal for 8b49a551695b0705f38348b99dc083b8: Waiting for close lock at 1732153543408Running coprocessor pre-close hooks at 1732153543408Disabling compacts and flushes for region at 1732153543408Disabling writes for close at 1732153543408Obtaining lock to block concurrent updates at 1732153543408Preparing flush snapshotting stores in 8b49a551695b0705f38348b99dc083b8 at 1732153543408Finished memstore snapshotting TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732153543408Flushing stores of TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8. at 1732153543409 (+1 ms)Flushing 8b49a551695b0705f38348b99dc083b8/A: creating writer at 1732153543409Flushing mob file 8b49a551695b0705f38348b99dc083b8/A: appending metadata at 1732153543414 (+5 ms)Flushing mob file 8b49a551695b0705f38348b99dc083b8/A: closing flushed file at 1732153543414Flushing 8b49a551695b0705f38348b99dc083b8/A: appending metadata at 1732153543823 (+409 ms)Flushing 8b49a551695b0705f38348b99dc083b8/A: closing flushed file at 1732153543823Flushing 8b49a551695b0705f38348b99dc083b8/B: creating writer at 1732153544231 (+408 ms)Flushing 8b49a551695b0705f38348b99dc083b8/B: appending metadata at 1732153544233 (+2 ms)Flushing 8b49a551695b0705f38348b99dc083b8/B: closing flushed file at 1732153544233Flushing 8b49a551695b0705f38348b99dc083b8/C: creating writer at 1732153544641 (+408 ms)Flushing 8b49a551695b0705f38348b99dc083b8/C: appending metadata at 1732153544644 (+3 ms)Flushing 8b49a551695b0705f38348b99dc083b8/C: closing flushed file at 1732153544644Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62169d7a: reopening flushed file at 1732153545053 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28e6e4ef: reopening flushed file at 1732153545057 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11791dad: reopening flushed file at 1732153545062 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 8b49a551695b0705f38348b99dc083b8 in 1658ms, sequenceid=340, compaction requested=false at 1732153545066 (+4 ms)Writing region close event to WAL at 1732153545134 (+68 ms)Running coprocessor post-close hooks at 1732153545136 (+2 ms)Closed at 1732153545136 2024-11-21T01:45:45,138 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] handler.UnassignRegionHandler(157): Closed 8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:45,139 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=8b49a551695b0705f38348b99dc083b8, regionState=CLOSED 2024-11-21T01:45:45,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=124, ppid=123, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 because future has completed 2024-11-21T01:45:45,142 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-21T01:45:45,142 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; CloseRegionProcedure 8b49a551695b0705f38348b99dc083b8, server=b93ada835ef8,41129,1732153434359 in 1.8860 sec 2024-11-21T01:45:45,143 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=123, resume processing ppid=122 2024-11-21T01:45:45,143 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, ppid=122, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b49a551695b0705f38348b99dc083b8, UNASSIGN in 1.8900 sec 2024-11-21T01:45:45,146 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-21T01:45:45,146 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8930 sec 2024-11-21T01:45:45,147 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732153545146"}]},"ts":"1732153545146"} 2024-11-21T01:45:45,148 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T01:45:45,148 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T01:45:45,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9020 sec 2024-11-21T01:45:45,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T01:45:45,393 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:45:45,394 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-21T01:45:45,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:45:45,395 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=125, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:45:45,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T01:45:45,396 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=125, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:45:45,400 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:45,402 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A, FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B, FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C, FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/recovered.edits] 2024-11-21T01:45:45,404 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/09399aa708104c2d9bedbdf647be6cbe to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/09399aa708104c2d9bedbdf647be6cbe 2024-11-21T01:45:45,405 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/0d14237613ab40e28cf8852bf9945e2a to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/A/0d14237613ab40e28cf8852bf9945e2a 2024-11-21T01:45:45,407 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/518badf0b52445868d601c69d8a7bd8e to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/518badf0b52445868d601c69d8a7bd8e 2024-11-21T01:45:45,408 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/6446a125734c46a49c0b65cf41f39f76 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/B/6446a125734c46a49c0b65cf41f39f76 2024-11-21T01:45:45,410 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/a102a4e7b08b4916a6d8e1f3f46d3d26 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/a102a4e7b08b4916a6d8e1f3f46d3d26 2024-11-21T01:45:45,411 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/cb407feb1af2490194752531d359eb79 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/C/cb407feb1af2490194752531d359eb79 2024-11-21T01:45:45,413 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/recovered.edits/343.seqid to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8/recovered.edits/343.seqid 2024-11-21T01:45:45,413 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:45,413 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T01:45:45,414 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T01:45:45,414 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-21T01:45:45,416 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210ab68cf9c51041c3a4461bda83584646_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210ab68cf9c51041c3a4461bda83584646_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:45,417 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211982ae7cb8b24eb5bbb36b026b987df3_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211982ae7cb8b24eb5bbb36b026b987df3_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:45,418 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211e220aae9a7f4404839ddb3ec15be172_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211e220aae9a7f4404839ddb3ec15be172_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:45,419 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212b704011824541f5a4010f7315788849_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212b704011824541f5a4010f7315788849_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:45,419 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212c911e590909434ba4feb45e37ac022e_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212c911e590909434ba4feb45e37ac022e_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:45,420 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112158ed9ceccddf4564b60873e27a1304b5_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112158ed9ceccddf4564b60873e27a1304b5_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:45,421 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112168f4102588934a65b6c73d6fc9c379e0_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112168f4102588934a65b6c73d6fc9c379e0_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:45,422 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216a31aa9fade349eea24b0574855cb98b_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216a31aa9fade349eea24b0574855cb98b_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:45,423 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216a42d998d6204ef5b6640c1992dfd700_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216a42d998d6204ef5b6640c1992dfd700_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:45,424 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217c7187638b6f4a74b13b87ef2ed34c6c_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217c7187638b6f4a74b13b87ef2ed34c6c_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:45,425 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411218b437536332e4692900f45323c3f2825_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411218b437536332e4692900f45323c3f2825_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:45,426 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219fc01a591ebe4cb09058581a17862423_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219fc01a591ebe4cb09058581a17862423_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:45,426 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a2cff75a73b4465ca7bd1f62d99f2dcd_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a2cff75a73b4465ca7bd1f62d99f2dcd_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:45,428 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b539400cb2ac405281a36c057eb1b386_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b539400cb2ac405281a36c057eb1b386_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:45,429 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c2415cb6986e48799c2a5501639e3e7f_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c2415cb6986e48799c2a5501639e3e7f_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:45,430 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e42cf4da458b4aeda4dc82f536de572f_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e42cf4da458b4aeda4dc82f536de572f_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:45,431 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121eab8cf561b9d4e108a1feca826a4c0ca_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121eab8cf561b9d4e108a1feca826a4c0ca_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:45,432 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ecb167b7b0b743b4aff95e2c51e47a07_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ecb167b7b0b743b4aff95e2c51e47a07_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:45,433 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ffb6a5318b914c45bf48e59937d26088_8b49a551695b0705f38348b99dc083b8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ffb6a5318b914c45bf48e59937d26088_8b49a551695b0705f38348b99dc083b8 2024-11-21T01:45:45,433 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T01:45:45,435 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=125, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:45:45,437 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T01:45:45,439 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T01:45:45,440 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=125, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:45:45,440 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T01:45:45,440 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732153545440"}]},"ts":"9223372036854775807"} 2024-11-21T01:45:45,442 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T01:45:45,442 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8b49a551695b0705f38348b99dc083b8, NAME => 'TestAcidGuarantees,,1732153516201.8b49a551695b0705f38348b99dc083b8.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T01:45:45,442 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T01:45:45,442 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732153545442"}]},"ts":"9223372036854775807"} 2024-11-21T01:45:45,443 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T01:45:45,444 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=125, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:45:45,444 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 50 msec 2024-11-21T01:45:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T01:45:45,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T01:45:45,503 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:45:45,513 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250 (was 252), OpenFileDescriptor=451 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=577 (was 590), ProcessCount=11 (was 11), AvailableMemoryMB=2410 (was 2489) 2024-11-21T01:45:45,521 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=577, ProcessCount=11, AvailableMemoryMB=2409 2024-11-21T01:45:45,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T01:45:45,522 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T01:45:45,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=126, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T01:45:45,524 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T01:45:45,524 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:45:45,524 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 126 2024-11-21T01:45:45,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-21T01:45:45,525 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T01:45:45,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742285_1461 (size=1061) 2024-11-21T01:45:45,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-21T01:45:45,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-21T01:45:45,933 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d 2024-11-21T01:45:45,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742286_1462 (size=53) 2024-11-21T01:45:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-21T01:45:46,348 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:45:46,348 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 7fd9dee8699da4e84c478aa5cfe68216, disabling compactions & flushes 2024-11-21T01:45:46,348 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:46,348 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:46,348 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. after waiting 0 ms 2024-11-21T01:45:46,348 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:46,348 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:46,348 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7fd9dee8699da4e84c478aa5cfe68216: Waiting for close lock at 1732153546348Disabling compacts and flushes for region at 1732153546348Disabling writes for close at 1732153546348Writing region close event to WAL at 1732153546348Closed at 1732153546348 2024-11-21T01:45:46,349 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T01:45:46,350 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732153546349"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732153546349"}]},"ts":"1732153546349"} 2024-11-21T01:45:46,351 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T01:45:46,352 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T01:45:46,352 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732153546352"}]},"ts":"1732153546352"} 2024-11-21T01:45:46,353 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T01:45:46,354 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fd9dee8699da4e84c478aa5cfe68216, ASSIGN}] 2024-11-21T01:45:46,355 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fd9dee8699da4e84c478aa5cfe68216, ASSIGN 2024-11-21T01:45:46,355 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fd9dee8699da4e84c478aa5cfe68216, ASSIGN; state=OFFLINE, location=b93ada835ef8,41129,1732153434359; forceNewPlan=false, retain=false 2024-11-21T01:45:46,506 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=127 updating hbase:meta row=7fd9dee8699da4e84c478aa5cfe68216, regionState=OPENING, regionLocation=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:46,508 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fd9dee8699da4e84c478aa5cfe68216, ASSIGN because future has completed 2024-11-21T01:45:46,508 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359}] 2024-11-21T01:45:46,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-21T01:45:46,664 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:46,664 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(7752): Opening region: {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} 2024-11-21T01:45:46,665 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:45:46,665 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:45:46,665 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(7794): checking encryption for 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:45:46,665 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(7797): checking classloading for 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:45:46,667 INFO [StoreOpener-7fd9dee8699da4e84c478aa5cfe68216-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:45:46,668 INFO [StoreOpener-7fd9dee8699da4e84c478aa5cfe68216-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T01:45:46,668 INFO [StoreOpener-7fd9dee8699da4e84c478aa5cfe68216-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7fd9dee8699da4e84c478aa5cfe68216 columnFamilyName A 2024-11-21T01:45:46,668 DEBUG [StoreOpener-7fd9dee8699da4e84c478aa5cfe68216-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:45:46,669 INFO [StoreOpener-7fd9dee8699da4e84c478aa5cfe68216-1 {}] regionserver.HStore(327): Store=7fd9dee8699da4e84c478aa5cfe68216/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:45:46,669 INFO [StoreOpener-7fd9dee8699da4e84c478aa5cfe68216-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:45:46,670 INFO [StoreOpener-7fd9dee8699da4e84c478aa5cfe68216-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T01:45:46,670 INFO [StoreOpener-7fd9dee8699da4e84c478aa5cfe68216-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7fd9dee8699da4e84c478aa5cfe68216 columnFamilyName B 2024-11-21T01:45:46,670 DEBUG [StoreOpener-7fd9dee8699da4e84c478aa5cfe68216-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:45:46,671 INFO [StoreOpener-7fd9dee8699da4e84c478aa5cfe68216-1 {}] regionserver.HStore(327): Store=7fd9dee8699da4e84c478aa5cfe68216/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:45:46,671 INFO [StoreOpener-7fd9dee8699da4e84c478aa5cfe68216-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:45:46,671 INFO [StoreOpener-7fd9dee8699da4e84c478aa5cfe68216-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T01:45:46,672 INFO [StoreOpener-7fd9dee8699da4e84c478aa5cfe68216-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7fd9dee8699da4e84c478aa5cfe68216 columnFamilyName C 2024-11-21T01:45:46,672 DEBUG [StoreOpener-7fd9dee8699da4e84c478aa5cfe68216-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:45:46,672 INFO [StoreOpener-7fd9dee8699da4e84c478aa5cfe68216-1 {}] regionserver.HStore(327): Store=7fd9dee8699da4e84c478aa5cfe68216/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:45:46,672 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:46,672 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1038): replaying wal for 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:45:46,673 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:45:46,673 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:45:46,674 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1048): stopping wal replay for 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:45:46,674 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1060): Cleaning up temporary data for 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:45:46,674 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T01:45:46,675 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1093): writing seq id for 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:45:46,677 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T01:45:46,677 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1114): Opened 7fd9dee8699da4e84c478aa5cfe68216; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63720855, jitterRate=-0.050485268235206604}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T01:45:46,677 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:45:46,678 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1006): Region open journal for 7fd9dee8699da4e84c478aa5cfe68216: Running coprocessor pre-open hook at 1732153546666Writing region info on filesystem at 1732153546666Initializing all the Stores at 1732153546666Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732153546666Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732153546666Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732153546666Cleaning up temporary data from old regions at 1732153546674 (+8 ms)Running coprocessor post-open hooks at 1732153546677 (+3 ms)Region opened successfully at 1732153546678 (+1 ms) 2024-11-21T01:45:46,679 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., pid=128, masterSystemTime=1732153546660 2024-11-21T01:45:46,680 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:46,680 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:46,681 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=127 updating hbase:meta row=7fd9dee8699da4e84c478aa5cfe68216, regionState=OPEN, openSeqNum=2, regionLocation=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:46,683 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=128, ppid=127, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 because future has completed 2024-11-21T01:45:46,686 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-21T01:45:46,686 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; OpenRegionProcedure 7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 in 176 msec 2024-11-21T01:45:46,688 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=127, resume processing ppid=126 2024-11-21T01:45:46,688 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, ppid=126, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fd9dee8699da4e84c478aa5cfe68216, ASSIGN in 332 msec 2024-11-21T01:45:46,689 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T01:45:46,689 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732153546689"}]},"ts":"1732153546689"} 2024-11-21T01:45:46,690 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T01:45:46,691 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T01:45:46,693 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1690 sec 2024-11-21T01:45:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-21T01:45:47,673 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:45:47,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10ca6048, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,674 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:45:47,674 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:45:47,675 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:45:47,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:45:47,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:45:47,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29be530e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:45:47,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:45:47,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:47,677 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58512, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:45:47,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c2fedb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,678 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:45:47,678 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:45:47,679 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:45:47,680 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49894, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:45:47,680 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T01:45:47,681 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is b93ada835ef8,36607,1732153433607 2024-11-21T01:45:47,681 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6009ab1 2024-11-21T01:45:47,681 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T01:45:47,682 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58524, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T01:45:47,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21ba7a64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,683 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:45:47,683 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:45:47,684 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:45:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:45:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:45:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27b6a6a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:45:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:45:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:47,685 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58546, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:45:47,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cbe5006, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,686 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff0b89c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,686 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:45:47,686 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:45:47,687 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:45:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:45:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:45:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@530d8586, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:45:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:45:47,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:47,688 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:45:47,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6507f43d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,689 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e5b9ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,689 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:45:47,689 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:45:47,690 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:45:47,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:45:47,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:45:47,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@376210c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:45:47,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:45:47,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:47,691 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58582, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:45:47,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6ef643, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5705feac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,692 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:45:47,692 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:45:47,693 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:45:47,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:45:47,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:45:47,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672b4013, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:45:47,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:45:47,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:47,694 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58604, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:45:47,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dfa7dde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,695 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a0ba1a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,695 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:45:47,695 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:45:47,696 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:45:47,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:45:47,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:45:47,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ffdff3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:45:47,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:45:47,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:47,697 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58612, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:45:47,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6877ef77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e9d7bc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,698 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:45:47,698 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:45:47,699 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:45:47,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:45:47,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:45:47,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c50b69c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:45:47,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:45:47,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:47,700 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:45:47,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46315c48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,701 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a625587, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,701 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:45:47,701 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:45:47,702 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:45:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:45:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:45:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c35a4a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:45:47,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:45:47,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:47,703 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58650, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:45:47,704 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6930fbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70fec919, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,705 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:45:47,705 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:45:47,705 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:45:47,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:45:47,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:45:47,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f2d8294, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:45:47,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:45:47,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:47,706 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58672, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:45:47,707 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45ecc281, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,707 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75f32ed9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,707 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:45:47,707 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:45:47,708 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:45:47,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:45:47,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:45:47,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a31f64d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:45:47,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:45:47,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:47,709 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58688, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:45:47,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a16a0c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,713 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66c63c4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,713 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:45:47,713 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:45:47,714 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:45:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:45:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:45:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dc5f6d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:45:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:45:47,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:45:47,715 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58704, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:45:47,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa78e08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:45:47,716 DEBUG [Thread-2060 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:45:47,716 DEBUG [Thread-2071 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:45:47,716 DEBUG [Thread-2068 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:45:47,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:45:47,717 DEBUG [Thread-2075 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:45:47,717 DEBUG [Thread-2066 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:45:47,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:45:47,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:45:47,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:45:47,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:45:47,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:45:47,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:45:47,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:45:47,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:45:47,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:45:47,718 DEBUG [Thread-2077 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:45:47,718 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:45:47,719 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:45:47,719 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49930, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:45:47,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:45:47,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:45:47,719 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49908, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:45:47,720 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49946, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:45:47,720 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49962, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:45:47,720 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:45:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-21T01:45:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T01:45:47,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2] 2024-11-21T01:45:47,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2] 2024-11-21T01:45:47,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2] 2024-11-21T01:45:47,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2] 2024-11-21T01:45:47,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2] 2024-11-21T01:45:47,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2] 2024-11-21T01:45:47,724 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:45:47,725 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:45:47,725 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:45:47,727 DEBUG [Thread-2079 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:45:47,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:45:47,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:45:47,730 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49980, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:45:47,730 DEBUG [Thread-2062 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:45:47,730 DEBUG [Thread-2073 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:45:47,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2] 2024-11-21T01:45:47,731 DEBUG [Thread-2064 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:45:47,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:45:47,732 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:45:47,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:45:47,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:45:47,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:45:47,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:45:47,734 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:45:47,734 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49988, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:45:47,735 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:45:47,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2] 2024-11-21T01:45:47,736 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2] 2024-11-21T01:45:47,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2] 2024-11-21T01:45:47,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fd9dee8699da4e84c478aa5cfe68216 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T01:45:47,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=A 2024-11-21T01:45:47,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:45:47,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=B 2024-11-21T01:45:47,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:47,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=C 2024-11-21T01:45:47,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:47,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/4682bcb408fe4ecb9d378a57773e8e05 is 50, key is test_row_0/A:col10/1732153547739/Put/seqid=0 2024-11-21T01:45:47,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732153547790, max timestamp=1732153547805Num uniques 859; 2024-11-21T01:45:47,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T01:45:47,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732153547790, max timestamp=1732153547808Num uniques 859; 2024-11-21T01:45:47,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T01:45:47,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732153547790, max timestamp=1732153547808Num uniques 859; 2024-11-21T01:45:47,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T01:45:47,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742287_1463 (size=14341) 2024-11-21T01:45:47,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/4682bcb408fe4ecb9d378a57773e8e05 2024-11-21T01:45:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T01:45:47,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/274ee6cdee0f41d79def78c75cca2d99 is 50, key is test_row_0/B:col10/1732153547739/Put/seqid=0 2024-11-21T01:45:47,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742288_1464 (size=12001) 2024-11-21T01:45:47,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T01:45:47,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:47,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:45:47,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:47,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:47,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:47,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:47,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:47,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153607887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:47,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:47,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:47,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:47,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:47,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153607900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:47,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:47,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:47,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:47,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:47,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153607901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:47,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:47,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:47,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:47,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153607903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:47,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:47,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:47,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:47,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:47,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:47,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153607905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:47,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:47,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:47,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:48,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:48,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153608013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:48,017 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:48,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:48,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153608013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:48,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T01:45:48,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T01:45:48,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:48,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:45:48,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:48,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:48,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:48,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:48,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:48,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153608033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:48,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:48,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:48,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:48,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:48,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153608033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:48,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:48,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:48,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:48,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:48,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153608033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:48,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:48,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:48,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:48,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T01:45:48,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:48,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:45:48,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:48,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:48,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:48,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:48,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:48,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153608223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:48,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:48,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153608223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:48,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:48,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:48,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:48,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:48,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:48,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:48,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/274ee6cdee0f41d79def78c75cca2d99 2024-11-21T01:45:48,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:48,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153608252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:48,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:48,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153608253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:48,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:48,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:48,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:48,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:48,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153608253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:48,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/ae2427a4441140f7ada3260d01368015 is 50, key is test_row_0/C:col10/1732153547739/Put/seqid=0 2024-11-21T01:45:48,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742289_1465 (size=12001) 2024-11-21T01:45:48,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T01:45:48,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T01:45:48,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:48,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:45:48,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:48,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:48,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:48,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:48,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T01:45:48,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:48,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:45:48,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:48,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:48,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:48,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:48,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153608561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:48,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:48,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:48,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:48,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:48,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153608561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:48,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:48,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153608574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:48,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:48,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153608574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:48,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:48,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153608577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:48,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:48,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:48,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:48,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T01:45:48,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:48,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:45:48,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:48,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:48,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:48,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:48,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/ae2427a4441140f7ada3260d01368015 2024-11-21T01:45:48,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/4682bcb408fe4ecb9d378a57773e8e05 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/4682bcb408fe4ecb9d378a57773e8e05 2024-11-21T01:45:48,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/4682bcb408fe4ecb9d378a57773e8e05, entries=200, sequenceid=12, filesize=14.0 K 2024-11-21T01:45:48,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/274ee6cdee0f41d79def78c75cca2d99 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/274ee6cdee0f41d79def78c75cca2d99 2024-11-21T01:45:48,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/274ee6cdee0f41d79def78c75cca2d99, entries=150, sequenceid=12, filesize=11.7 K 2024-11-21T01:45:48,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/ae2427a4441140f7ada3260d01368015 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/ae2427a4441140f7ada3260d01368015 2024-11-21T01:45:48,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/ae2427a4441140f7ada3260d01368015, entries=150, sequenceid=12, filesize=11.7 K 2024-11-21T01:45:48,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 7fd9dee8699da4e84c478aa5cfe68216 in 983ms, sequenceid=12, compaction requested=false 2024-11-21T01:45:48,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:45:48,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T01:45:48,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:48,821 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 7fd9dee8699da4e84c478aa5cfe68216 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-21T01:45:48,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=A 2024-11-21T01:45:48,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:45:48,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=B 2024-11-21T01:45:48,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:45:48,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=C 2024-11-21T01:45:48,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:45:48,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/a5b1ac8a547d403a90979cc9be63246b is 50, key is test_row_0/A:col10/1732153547902/Put/seqid=0 2024-11-21T01:45:48,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742290_1466 (size=12001) 2024-11-21T01:45:48,835 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/a5b1ac8a547d403a90979cc9be63246b 2024-11-21T01:45:48,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/300e254c1d854d359b3fb9d6ad0586bd is 50, key is test_row_0/B:col10/1732153547902/Put/seqid=0 2024-11-21T01:45:48,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T01:45:48,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742291_1467 (size=12001) 2024-11-21T01:45:48,866 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/300e254c1d854d359b3fb9d6ad0586bd 2024-11-21T01:45:48,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/395b3333fbeb402bb1211e1e4faf5c5b is 50, key is test_row_0/C:col10/1732153547902/Put/seqid=0 2024-11-21T01:45:48,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742292_1468 (size=12001) 2024-11-21T01:45:49,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:45:49,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:45:49,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:49,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153609104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:49,109 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,109 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,109 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:49,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:49,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153609105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:49,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:49,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153609107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:49,122 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,122 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:49,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:49,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:49,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153609107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:49,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:49,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:49,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153609108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:49,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:49,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:49,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153609213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:49,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:49,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:49,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153609233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:49,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:49,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153609234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:49,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:49,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:49,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:49,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153609234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:49,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:49,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:49,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153609234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:49,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:49,298 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/395b3333fbeb402bb1211e1e4faf5c5b 2024-11-21T01:45:49,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/a5b1ac8a547d403a90979cc9be63246b as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/a5b1ac8a547d403a90979cc9be63246b 2024-11-21T01:45:49,305 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/a5b1ac8a547d403a90979cc9be63246b, entries=150, sequenceid=48, filesize=11.7 K 2024-11-21T01:45:49,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/300e254c1d854d359b3fb9d6ad0586bd as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/300e254c1d854d359b3fb9d6ad0586bd 2024-11-21T01:45:49,310 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/300e254c1d854d359b3fb9d6ad0586bd, entries=150, sequenceid=48, filesize=11.7 K 2024-11-21T01:45:49,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/395b3333fbeb402bb1211e1e4faf5c5b as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/395b3333fbeb402bb1211e1e4faf5c5b 2024-11-21T01:45:49,313 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/395b3333fbeb402bb1211e1e4faf5c5b, entries=150, sequenceid=48, filesize=11.7 K 2024-11-21T01:45:49,314 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=60.38 KB/61830 for 7fd9dee8699da4e84c478aa5cfe68216 in 493ms, sequenceid=48, compaction requested=false 2024-11-21T01:45:49,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:45:49,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:49,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-21T01:45:49,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-21T01:45:49,318 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-21T01:45:49,318 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5910 sec 2024-11-21T01:45:49,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.5990 sec 2024-11-21T01:45:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:45:49,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fd9dee8699da4e84c478aa5cfe68216 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T01:45:49,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=A 2024-11-21T01:45:49,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:49,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=B 2024-11-21T01:45:49,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:49,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=C 2024-11-21T01:45:49,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:49,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/0e2fbdc9682f44ce8aa57ed5ca020d31 is 50, key is test_row_0/A:col10/1732153549438/Put/seqid=0 2024-11-21T01:45:49,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742293_1469 (size=14341) 2024-11-21T01:45:49,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/0e2fbdc9682f44ce8aa57ed5ca020d31 2024-11-21T01:45:49,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/5c956128cf96498382d3e1dac0cc8a64 is 50, key is test_row_0/B:col10/1732153549438/Put/seqid=0 2024-11-21T01:45:49,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742294_1470 (size=12001) 2024-11-21T01:45:49,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:49,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153609492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:49,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:49,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:49,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153609493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:49,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:49,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:49,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153609497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:49,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:49,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:49,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153609501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:49,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:49,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:49,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153609502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:49,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:49,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153609623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:49,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153609623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153609623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:49,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:49,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153609623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:49,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:49,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:49,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:49,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153609623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:49,634 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,634 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,634 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:49,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:49,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153609843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:49,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:49,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153609844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:49,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:49,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153609844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:49,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:49,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:49,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153609844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:49,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:49,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:49,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153609845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:49,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:49,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:49,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:49,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T01:45:49,863 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:45:49,864 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:45:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-21T01:45:49,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T01:45:49,868 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:45:49,869 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:45:49,869 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:45:49,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/5c956128cf96498382d3e1dac0cc8a64 2024-11-21T01:45:49,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/974d1cdfe19d49a39113a705b80de60e is 50, key is test_row_0/C:col10/1732153549438/Put/seqid=0 2024-11-21T01:45:49,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742295_1471 (size=12001) 2024-11-21T01:45:49,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/974d1cdfe19d49a39113a705b80de60e 2024-11-21T01:45:49,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/0e2fbdc9682f44ce8aa57ed5ca020d31 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/0e2fbdc9682f44ce8aa57ed5ca020d31 2024-11-21T01:45:49,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/0e2fbdc9682f44ce8aa57ed5ca020d31, entries=200, sequenceid=61, filesize=14.0 K 2024-11-21T01:45:49,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/5c956128cf96498382d3e1dac0cc8a64 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/5c956128cf96498382d3e1dac0cc8a64 2024-11-21T01:45:49,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/5c956128cf96498382d3e1dac0cc8a64, entries=150, sequenceid=61, filesize=11.7 K 2024-11-21T01:45:49,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T01:45:49,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/974d1cdfe19d49a39113a705b80de60e as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/974d1cdfe19d49a39113a705b80de60e 2024-11-21T01:45:50,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/974d1cdfe19d49a39113a705b80de60e, entries=150, sequenceid=61, filesize=11.7 K 2024-11-21T01:45:50,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 7fd9dee8699da4e84c478aa5cfe68216 in 567ms, sequenceid=61, compaction requested=true 2024-11-21T01:45:50,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:45:50,006 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:50,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fd9dee8699da4e84c478aa5cfe68216:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:45:50,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:50,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fd9dee8699da4e84c478aa5cfe68216:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:45:50,007 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:50,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:50,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fd9dee8699da4e84c478aa5cfe68216:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:45:50,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:50,011 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:50,011 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 7fd9dee8699da4e84c478aa5cfe68216/A is initiating minor compaction (all files) 2024-11-21T01:45:50,011 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fd9dee8699da4e84c478aa5cfe68216/A in TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:50,012 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/4682bcb408fe4ecb9d378a57773e8e05, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/a5b1ac8a547d403a90979cc9be63246b, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/0e2fbdc9682f44ce8aa57ed5ca020d31] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp, totalSize=39.7 K 2024-11-21T01:45:50,012 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:50,012 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 7fd9dee8699da4e84c478aa5cfe68216/B is initiating minor compaction (all files) 2024-11-21T01:45:50,012 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fd9dee8699da4e84c478aa5cfe68216/B in TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:50,012 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/274ee6cdee0f41d79def78c75cca2d99, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/300e254c1d854d359b3fb9d6ad0586bd, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/5c956128cf96498382d3e1dac0cc8a64] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp, totalSize=35.2 K 2024-11-21T01:45:50,012 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4682bcb408fe4ecb9d378a57773e8e05, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732153547725 2024-11-21T01:45:50,013 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 274ee6cdee0f41d79def78c75cca2d99, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732153547731 2024-11-21T01:45:50,013 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5b1ac8a547d403a90979cc9be63246b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732153547899 2024-11-21T01:45:50,013 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 300e254c1d854d359b3fb9d6ad0586bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732153547899 2024-11-21T01:45:50,013 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c956128cf96498382d3e1dac0cc8a64, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732153549105 2024-11-21T01:45:50,013 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e2fbdc9682f44ce8aa57ed5ca020d31, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732153549105 2024-11-21T01:45:50,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-21T01:45:50,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:50,022 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 7fd9dee8699da4e84c478aa5cfe68216 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T01:45:50,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=A 2024-11-21T01:45:50,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:50,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=B 2024-11-21T01:45:50,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:50,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=C 2024-11-21T01:45:50,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:50,023 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fd9dee8699da4e84c478aa5cfe68216#A#compaction#396 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:50,024 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/978e9238355741b2adc373ced769e159 is 50, key is test_row_0/A:col10/1732153549438/Put/seqid=0 2024-11-21T01:45:50,026 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fd9dee8699da4e84c478aa5cfe68216#B#compaction#397 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:50,027 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/cb8e8c2bf78841aeb5ba833eabe1c9dc is 50, key is test_row_0/B:col10/1732153549438/Put/seqid=0 2024-11-21T01:45:50,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/3f9a0a57cf784c139419509937ecbf8d is 50, key is test_row_0/A:col10/1732153549484/Put/seqid=0 2024-11-21T01:45:50,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742296_1472 (size=12104) 2024-11-21T01:45:50,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742298_1474 (size=12001) 2024-11-21T01:45:50,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742297_1473 (size=12104) 2024-11-21T01:45:50,107 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/978e9238355741b2adc373ced769e159 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/978e9238355741b2adc373ced769e159 2024-11-21T01:45:50,108 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/3f9a0a57cf784c139419509937ecbf8d 2024-11-21T01:45:50,118 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/cb8e8c2bf78841aeb5ba833eabe1c9dc as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/cb8e8c2bf78841aeb5ba833eabe1c9dc 2024-11-21T01:45:50,124 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fd9dee8699da4e84c478aa5cfe68216/A of 7fd9dee8699da4e84c478aa5cfe68216 into 978e9238355741b2adc373ced769e159(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:50,124 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:45:50,124 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., storeName=7fd9dee8699da4e84c478aa5cfe68216/A, priority=13, startTime=1732153550006; duration=0sec 2024-11-21T01:45:50,125 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:50,125 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fd9dee8699da4e84c478aa5cfe68216:A 2024-11-21T01:45:50,125 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:50,128 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:50,128 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 7fd9dee8699da4e84c478aa5cfe68216/C is initiating minor compaction (all files) 2024-11-21T01:45:50,128 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fd9dee8699da4e84c478aa5cfe68216/B of 7fd9dee8699da4e84c478aa5cfe68216 into cb8e8c2bf78841aeb5ba833eabe1c9dc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:50,128 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fd9dee8699da4e84c478aa5cfe68216/C in TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:50,128 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:45:50,128 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., storeName=7fd9dee8699da4e84c478aa5cfe68216/B, priority=13, startTime=1732153550007; duration=0sec 2024-11-21T01:45:50,128 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/ae2427a4441140f7ada3260d01368015, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/395b3333fbeb402bb1211e1e4faf5c5b, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/974d1cdfe19d49a39113a705b80de60e] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp, totalSize=35.2 K 2024-11-21T01:45:50,128 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:50,128 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fd9dee8699da4e84c478aa5cfe68216:B 2024-11-21T01:45:50,133 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae2427a4441140f7ada3260d01368015, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732153547731 2024-11-21T01:45:50,135 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 395b3333fbeb402bb1211e1e4faf5c5b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732153547899 2024-11-21T01:45:50,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/aa132045d940464fb6a1accaee84625a is 50, key is test_row_0/B:col10/1732153549484/Put/seqid=0 2024-11-21T01:45:50,138 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 974d1cdfe19d49a39113a705b80de60e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732153549105 2024-11-21T01:45:50,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:45:50,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:45:50,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742299_1475 (size=12001) 2024-11-21T01:45:50,173 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fd9dee8699da4e84c478aa5cfe68216#C#compaction#400 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:50,174 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/c6abfbfcfa5147f9871956523f58cc26 is 50, key is test_row_0/C:col10/1732153549438/Put/seqid=0 2024-11-21T01:45:50,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T01:45:50,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742300_1476 (size=12104) 2024-11-21T01:45:50,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:50,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153610179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:50,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:50,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:50,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153610186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:50,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153610186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:50,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:50,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153610186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:50,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:50,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153610198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:50,201 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,201 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,201 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:50,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:50,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:50,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153610302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:50,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:50,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153610303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:50,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153610303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:50,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153610303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:50,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:50,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:50,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153610303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:50,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:50,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T01:45:50,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:50,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153610522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:50,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153610522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:50,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:50,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153610522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:50,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153610523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:50,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153610523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:50,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:50,573 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/aa132045d940464fb6a1accaee84625a 2024-11-21T01:45:50,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/9b97dcc6b60745c18d048209821e28eb is 50, key is test_row_0/C:col10/1732153549484/Put/seqid=0 2024-11-21T01:45:50,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742301_1477 (size=12001) 2024-11-21T01:45:50,594 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/c6abfbfcfa5147f9871956523f58cc26 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/c6abfbfcfa5147f9871956523f58cc26 2024-11-21T01:45:50,599 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fd9dee8699da4e84c478aa5cfe68216/C of 7fd9dee8699da4e84c478aa5cfe68216 into c6abfbfcfa5147f9871956523f58cc26(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:50,599 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:45:50,599 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., storeName=7fd9dee8699da4e84c478aa5cfe68216/C, priority=13, startTime=1732153550007; duration=0sec 2024-11-21T01:45:50,599 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:50,599 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fd9dee8699da4e84c478aa5cfe68216:C 2024-11-21T01:45:50,666 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T01:45:50,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153610833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:50,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:50,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:50,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153610843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:50,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:50,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153610843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:50,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:50,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153610843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:50,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:50,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:50,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:50,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153610843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:50,985 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/9b97dcc6b60745c18d048209821e28eb 2024-11-21T01:45:50,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/3f9a0a57cf784c139419509937ecbf8d as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/3f9a0a57cf784c139419509937ecbf8d 2024-11-21T01:45:50,993 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/3f9a0a57cf784c139419509937ecbf8d, entries=150, sequenceid=80, filesize=11.7 K 2024-11-21T01:45:50,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/aa132045d940464fb6a1accaee84625a as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/aa132045d940464fb6a1accaee84625a 2024-11-21T01:45:50,997 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/aa132045d940464fb6a1accaee84625a, entries=150, sequenceid=80, filesize=11.7 K 2024-11-21T01:45:50,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/9b97dcc6b60745c18d048209821e28eb as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/9b97dcc6b60745c18d048209821e28eb 2024-11-21T01:45:51,001 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/9b97dcc6b60745c18d048209821e28eb, entries=150, sequenceid=80, filesize=11.7 K 2024-11-21T01:45:51,002 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 7fd9dee8699da4e84c478aa5cfe68216 in 980ms, sequenceid=80, compaction requested=false 2024-11-21T01:45:51,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:45:51,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:51,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-21T01:45:51,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T01:45:51,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-21T01:45:51,005 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-21T01:45:51,005 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1340 sec 2024-11-21T01:45:51,007 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.1420 sec 2024-11-21T01:45:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:45:51,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fd9dee8699da4e84c478aa5cfe68216 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T01:45:51,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=A 2024-11-21T01:45:51,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:51,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=B 2024-11-21T01:45:51,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:51,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=C 2024-11-21T01:45:51,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:51,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/73633fdda07e450ca9a018bd898522ae is 50, key is test_row_0/A:col10/1732153550179/Put/seqid=0 2024-11-21T01:45:51,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742302_1478 (size=14341) 2024-11-21T01:45:51,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153611406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:51,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:51,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:51,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:51,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153611418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:51,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:51,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:51,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153611420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:51,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:51,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:51,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:51,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:51,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153611420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:51,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153611420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:51,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:51,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:51,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:51,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:51,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:51,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:51,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153611523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:51,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:51,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:51,533 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:51,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153611543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:51,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:51,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:51,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:51,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153611543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:51,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:51,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:51,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:51,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153611543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:51,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:51,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:51,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:51,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153611543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:51,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:51,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:51,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:51,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153611743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:51,754 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:51,754 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:51,754 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:51,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153611753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:51,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153611753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:51,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153611753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:51,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153611763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:51,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:51,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:51,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:51,807 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/73633fdda07e450ca9a018bd898522ae 2024-11-21T01:45:51,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/75ea9367ec21478e946fc4a028123385 is 50, key is test_row_0/B:col10/1732153550179/Put/seqid=0 2024-11-21T01:45:51,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742303_1479 (size=12001) 2024-11-21T01:45:52,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T01:45:52,013 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:45:52,014 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:45:52,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-21T01:45:52,016 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:45:52,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T01:45:52,017 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:45:52,017 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:45:52,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:52,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153612064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:52,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:52,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:52,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153612064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:52,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:52,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:52,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153612073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:52,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:52,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153612073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:52,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:52,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:52,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:52,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153612084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:52,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:52,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T01:45:52,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-21T01:45:52,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:52,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:45:52,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:52,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:52,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:52,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/75ea9367ec21478e946fc4a028123385 2024-11-21T01:45:52,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/4824db2ed3a34d1aae63077d03ee41c9 is 50, key is test_row_0/C:col10/1732153550179/Put/seqid=0 2024-11-21T01:45:52,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742304_1480 (size=12001) 2024-11-21T01:45:52,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/4824db2ed3a34d1aae63077d03ee41c9 2024-11-21T01:45:52,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/73633fdda07e450ca9a018bd898522ae as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/73633fdda07e450ca9a018bd898522ae 2024-11-21T01:45:52,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/73633fdda07e450ca9a018bd898522ae, entries=200, sequenceid=97, filesize=14.0 K 2024-11-21T01:45:52,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/75ea9367ec21478e946fc4a028123385 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/75ea9367ec21478e946fc4a028123385 2024-11-21T01:45:52,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/75ea9367ec21478e946fc4a028123385, entries=150, sequenceid=97, filesize=11.7 K 2024-11-21T01:45:52,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/4824db2ed3a34d1aae63077d03ee41c9 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/4824db2ed3a34d1aae63077d03ee41c9 2024-11-21T01:45:52,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/4824db2ed3a34d1aae63077d03ee41c9, entries=150, sequenceid=97, filesize=11.7 K 2024-11-21T01:45:52,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 7fd9dee8699da4e84c478aa5cfe68216 in 903ms, sequenceid=97, compaction requested=true 2024-11-21T01:45:52,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:45:52,260 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:52,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fd9dee8699da4e84c478aa5cfe68216:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:45:52,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:52,261 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:52,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fd9dee8699da4e84c478aa5cfe68216:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:45:52,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:52,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fd9dee8699da4e84c478aa5cfe68216:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:45:52,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:52,262 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:52,262 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 7fd9dee8699da4e84c478aa5cfe68216/A is initiating minor compaction (all files) 2024-11-21T01:45:52,262 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fd9dee8699da4e84c478aa5cfe68216/A in TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:52,262 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/978e9238355741b2adc373ced769e159, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/3f9a0a57cf784c139419509937ecbf8d, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/73633fdda07e450ca9a018bd898522ae] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp, totalSize=37.5 K 2024-11-21T01:45:52,262 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:52,262 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 7fd9dee8699da4e84c478aa5cfe68216/B is initiating minor compaction (all files) 2024-11-21T01:45:52,262 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fd9dee8699da4e84c478aa5cfe68216/B in TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:52,262 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/cb8e8c2bf78841aeb5ba833eabe1c9dc, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/aa132045d940464fb6a1accaee84625a, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/75ea9367ec21478e946fc4a028123385] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp, totalSize=35.3 K 2024-11-21T01:45:52,262 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 978e9238355741b2adc373ced769e159, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732153549105 2024-11-21T01:45:52,262 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting cb8e8c2bf78841aeb5ba833eabe1c9dc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732153549105 2024-11-21T01:45:52,263 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting aa132045d940464fb6a1accaee84625a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732153549484 2024-11-21T01:45:52,263 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f9a0a57cf784c139419509937ecbf8d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732153549484 2024-11-21T01:45:52,263 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73633fdda07e450ca9a018bd898522ae, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732153550179 2024-11-21T01:45:52,263 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 75ea9367ec21478e946fc4a028123385, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732153550179 2024-11-21T01:45:52,271 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fd9dee8699da4e84c478aa5cfe68216#B#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:52,271 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/7ca412052eeb470bbecc09f447937138 is 50, key is test_row_0/B:col10/1732153550179/Put/seqid=0 2024-11-21T01:45:52,271 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fd9dee8699da4e84c478aa5cfe68216#A#compaction#406 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:52,272 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/914b9ce601b7490a84f8ec1c065c9342 is 50, key is test_row_0/A:col10/1732153550179/Put/seqid=0 2024-11-21T01:45:52,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742306_1482 (size=12207) 2024-11-21T01:45:52,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742305_1481 (size=12207) 2024-11-21T01:45:52,295 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/7ca412052eeb470bbecc09f447937138 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/7ca412052eeb470bbecc09f447937138 2024-11-21T01:45:52,306 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fd9dee8699da4e84c478aa5cfe68216/B of 7fd9dee8699da4e84c478aa5cfe68216 into 7ca412052eeb470bbecc09f447937138(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:52,306 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:45:52,306 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., storeName=7fd9dee8699da4e84c478aa5cfe68216/B, priority=13, startTime=1732153552261; duration=0sec 2024-11-21T01:45:52,306 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:52,306 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fd9dee8699da4e84c478aa5cfe68216:B 2024-11-21T01:45:52,306 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:52,307 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:52,307 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 7fd9dee8699da4e84c478aa5cfe68216/C is initiating minor compaction (all files) 2024-11-21T01:45:52,307 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fd9dee8699da4e84c478aa5cfe68216/C in TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:52,307 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/c6abfbfcfa5147f9871956523f58cc26, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/9b97dcc6b60745c18d048209821e28eb, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/4824db2ed3a34d1aae63077d03ee41c9] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp, totalSize=35.3 K 2024-11-21T01:45:52,308 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting c6abfbfcfa5147f9871956523f58cc26, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732153549105 2024-11-21T01:45:52,308 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b97dcc6b60745c18d048209821e28eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732153549484 2024-11-21T01:45:52,308 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 4824db2ed3a34d1aae63077d03ee41c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732153550179 2024-11-21T01:45:52,315 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fd9dee8699da4e84c478aa5cfe68216#C#compaction#407 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:52,316 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/cc6a3c30149f425c960e3cadaf80d867 is 50, key is test_row_0/C:col10/1732153550179/Put/seqid=0 2024-11-21T01:45:52,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-21T01:45:52,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:52,324 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 7fd9dee8699da4e84c478aa5cfe68216 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T01:45:52,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=A 2024-11-21T01:45:52,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:52,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=B 2024-11-21T01:45:52,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:52,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=C 2024-11-21T01:45:52,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:52,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742307_1483 (size=12207) 2024-11-21T01:45:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T01:45:52,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/aac946e8917d4ae69fa1b4cfc7e5280b is 50, key is test_row_0/A:col10/1732153551418/Put/seqid=0 2024-11-21T01:45:52,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742308_1484 (size=12001) 2024-11-21T01:45:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:45:52,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:45:52,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153612620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:52,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:52,631 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T01:45:52,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153612622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:52,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:52,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153612626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:52,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:52,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153612627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:52,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:52,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T01:45:52,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:52,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153612632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:52,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:52,700 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/914b9ce601b7490a84f8ec1c065c9342 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/914b9ce601b7490a84f8ec1c065c9342 2024-11-21T01:45:52,705 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fd9dee8699da4e84c478aa5cfe68216/A of 7fd9dee8699da4e84c478aa5cfe68216 into 914b9ce601b7490a84f8ec1c065c9342(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:52,705 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:45:52,706 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., storeName=7fd9dee8699da4e84c478aa5cfe68216/A, priority=13, startTime=1732153552260; duration=0sec 2024-11-21T01:45:52,706 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:52,706 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fd9dee8699da4e84c478aa5cfe68216:A 2024-11-21T01:45:52,734 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/cc6a3c30149f425c960e3cadaf80d867 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/cc6a3c30149f425c960e3cadaf80d867 2024-11-21T01:45:52,742 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fd9dee8699da4e84c478aa5cfe68216/C of 7fd9dee8699da4e84c478aa5cfe68216 into cc6a3c30149f425c960e3cadaf80d867(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:52,742 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:45:52,742 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., storeName=7fd9dee8699da4e84c478aa5cfe68216/C, priority=13, startTime=1732153552261; duration=0sec 2024-11-21T01:45:52,742 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:52,742 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fd9dee8699da4e84c478aa5cfe68216:C 2024-11-21T01:45:52,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:52,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153612736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:52,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:52,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:52,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153612743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:52,757 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/aac946e8917d4ae69fa1b4cfc7e5280b 2024-11-21T01:45:52,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:52,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:52,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153612757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:52,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:52,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:52,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153612757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:52,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:52,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153612764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:52,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:52,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/ea693fdba7904eec86e92512c16d02c8 is 50, key is test_row_0/B:col10/1732153551418/Put/seqid=0 2024-11-21T01:45:52,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742309_1485 (size=12001) 2024-11-21T01:45:52,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:52,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153612953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:52,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:52,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:52,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153612963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:52,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:52,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:52,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153612973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:52,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:52,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:52,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153612983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:52,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:52,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:52,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:52,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153612985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:53,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:53,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:53,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:53,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T01:45:53,243 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/ea693fdba7904eec86e92512c16d02c8 2024-11-21T01:45:53,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/be18d998e2b441c29912257acacd4967 is 50, key is test_row_0/C:col10/1732153551418/Put/seqid=0 2024-11-21T01:45:53,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742310_1486 (size=12001) 2024-11-21T01:45:53,260 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/be18d998e2b441c29912257acacd4967 2024-11-21T01:45:53,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/aac946e8917d4ae69fa1b4cfc7e5280b as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/aac946e8917d4ae69fa1b4cfc7e5280b 2024-11-21T01:45:53,277 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/aac946e8917d4ae69fa1b4cfc7e5280b, entries=150, sequenceid=117, filesize=11.7 K 2024-11-21T01:45:53,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/ea693fdba7904eec86e92512c16d02c8 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/ea693fdba7904eec86e92512c16d02c8 2024-11-21T01:45:53,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:53,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153613272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:53,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:53,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:53,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153613272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:53,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:53,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:53,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:53,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:53,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:53,287 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/ea693fdba7904eec86e92512c16d02c8, entries=150, sequenceid=117, filesize=11.7 K 2024-11-21T01:45:53,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/be18d998e2b441c29912257acacd4967 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/be18d998e2b441c29912257acacd4967 2024-11-21T01:45:53,301 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/be18d998e2b441c29912257acacd4967, entries=150, sequenceid=117, filesize=11.7 K 2024-11-21T01:45:53,303 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 7fd9dee8699da4e84c478aa5cfe68216 in 979ms, sequenceid=117, compaction requested=false 2024-11-21T01:45:53,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:45:53,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:53,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-21T01:45:53,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:45:53,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fd9dee8699da4e84c478aa5cfe68216 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T01:45:53,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-21T01:45:53,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=A 2024-11-21T01:45:53,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:53,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=B 2024-11-21T01:45:53,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:53,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=C 2024-11-21T01:45:53,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:53,312 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-21T01:45:53,312 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2880 sec 2024-11-21T01:45:53,315 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.2990 sec 2024-11-21T01:45:53,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/45c675a3512f4fcea05619bc5c3c0396 is 50, key is test_row_0/A:col10/1732153553301/Put/seqid=0 2024-11-21T01:45:53,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742311_1487 (size=14441) 2024-11-21T01:45:53,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:53,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153613357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:53,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:53,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:53,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:53,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:53,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153613381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:53,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:53,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153613385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:53,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:53,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:53,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:53,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:53,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:53,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:53,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:53,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153613503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:53,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:53,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:53,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:53,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:53,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153613503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:53,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:53,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153613512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:53,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/45c675a3512f4fcea05619bc5c3c0396 2024-11-21T01:45:53,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:53,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153613724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:53,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:53,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:53,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:53,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:53,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153613725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:53,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:53,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153613725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:53,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/4e1131e830714e11bae9a6380925af6a is 50, key is test_row_0/B:col10/1732153553301/Put/seqid=0 2024-11-21T01:45:53,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742312_1488 (size=12051) 2024-11-21T01:45:53,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/4e1131e830714e11bae9a6380925af6a 2024-11-21T01:45:53,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:53,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153613798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:53,813 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:53,813 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:53,813 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:53,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:53,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153613799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:53,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:53,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:53,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:53,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/0e425d310e254d2783ab4af61e135976 is 50, key is test_row_0/C:col10/1732153553301/Put/seqid=0 2024-11-21T01:45:53,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742313_1489 (size=12051) 2024-11-21T01:45:53,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/0e425d310e254d2783ab4af61e135976 2024-11-21T01:45:53,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/45c675a3512f4fcea05619bc5c3c0396 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/45c675a3512f4fcea05619bc5c3c0396 2024-11-21T01:45:53,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/45c675a3512f4fcea05619bc5c3c0396, entries=200, sequenceid=133, filesize=14.1 K 2024-11-21T01:45:53,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/4e1131e830714e11bae9a6380925af6a as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/4e1131e830714e11bae9a6380925af6a 2024-11-21T01:45:53,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/4e1131e830714e11bae9a6380925af6a, entries=150, sequenceid=133, filesize=11.8 K 2024-11-21T01:45:53,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/0e425d310e254d2783ab4af61e135976 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/0e425d310e254d2783ab4af61e135976 2024-11-21T01:45:53,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/0e425d310e254d2783ab4af61e135976, entries=150, sequenceid=133, filesize=11.8 K 2024-11-21T01:45:53,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 7fd9dee8699da4e84c478aa5cfe68216 in 665ms, sequenceid=133, compaction requested=true 2024-11-21T01:45:53,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:45:53,969 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:53,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fd9dee8699da4e84c478aa5cfe68216:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:45:53,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:53,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fd9dee8699da4e84c478aa5cfe68216:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:45:53,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:53,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fd9dee8699da4e84c478aa5cfe68216:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:45:53,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:53,970 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:53,977 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38649 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:53,977 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 7fd9dee8699da4e84c478aa5cfe68216/A is initiating minor compaction (all files) 2024-11-21T01:45:53,977 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fd9dee8699da4e84c478aa5cfe68216/A in TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:53,978 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/914b9ce601b7490a84f8ec1c065c9342, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/aac946e8917d4ae69fa1b4cfc7e5280b, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/45c675a3512f4fcea05619bc5c3c0396] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp, totalSize=37.7 K 2024-11-21T01:45:53,978 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:53,978 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 7fd9dee8699da4e84c478aa5cfe68216/B is initiating minor compaction (all files) 2024-11-21T01:45:53,978 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fd9dee8699da4e84c478aa5cfe68216/B in TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:53,978 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/7ca412052eeb470bbecc09f447937138, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/ea693fdba7904eec86e92512c16d02c8, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/4e1131e830714e11bae9a6380925af6a] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp, totalSize=35.4 K 2024-11-21T01:45:53,980 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 914b9ce601b7490a84f8ec1c065c9342, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732153550179 2024-11-21T01:45:53,980 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ca412052eeb470bbecc09f447937138, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732153550179 2024-11-21T01:45:53,983 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting aac946e8917d4ae69fa1b4cfc7e5280b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732153551413 2024-11-21T01:45:53,985 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting ea693fdba7904eec86e92512c16d02c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732153551413 2024-11-21T01:45:53,985 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45c675a3512f4fcea05619bc5c3c0396, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732153552620 2024-11-21T01:45:53,988 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e1131e830714e11bae9a6380925af6a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732153552620 2024-11-21T01:45:54,028 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fd9dee8699da4e84c478aa5cfe68216#A#compaction#414 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:54,028 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fd9dee8699da4e84c478aa5cfe68216#B#compaction#415 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:54,029 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/180751cadac140faaa1d2add6a1e33e5 is 50, key is test_row_0/A:col10/1732153553301/Put/seqid=0 2024-11-21T01:45:54,029 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/9e004e4e620a43fba94885919e1ae245 is 50, key is test_row_0/B:col10/1732153553301/Put/seqid=0 2024-11-21T01:45:54,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:45:54,055 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fd9dee8699da4e84c478aa5cfe68216 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T01:45:54,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=A 2024-11-21T01:45:54,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:54,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=B 2024-11-21T01:45:54,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:54,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=C 2024-11-21T01:45:54,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:54,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742314_1490 (size=12359) 2024-11-21T01:45:54,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742315_1491 (size=12359) 2024-11-21T01:45:54,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/2abb11d59d9f4ddbb70eaaaa21a28db2 is 50, key is test_row_0/A:col10/1732153553377/Put/seqid=0 2024-11-21T01:45:54,086 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/9e004e4e620a43fba94885919e1ae245 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/9e004e4e620a43fba94885919e1ae245 2024-11-21T01:45:54,092 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fd9dee8699da4e84c478aa5cfe68216/B of 7fd9dee8699da4e84c478aa5cfe68216 into 9e004e4e620a43fba94885919e1ae245(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:54,092 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:45:54,092 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., storeName=7fd9dee8699da4e84c478aa5cfe68216/B, priority=13, startTime=1732153553970; duration=0sec 2024-11-21T01:45:54,093 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:54,093 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fd9dee8699da4e84c478aa5cfe68216:B 2024-11-21T01:45:54,093 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:54,093 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:54,094 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 7fd9dee8699da4e84c478aa5cfe68216/C is initiating minor compaction (all files) 2024-11-21T01:45:54,094 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fd9dee8699da4e84c478aa5cfe68216/C in TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:54,094 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/cc6a3c30149f425c960e3cadaf80d867, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/be18d998e2b441c29912257acacd4967, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/0e425d310e254d2783ab4af61e135976] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp, totalSize=35.4 K 2024-11-21T01:45:54,094 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting cc6a3c30149f425c960e3cadaf80d867, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732153550179 2024-11-21T01:45:54,094 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting be18d998e2b441c29912257acacd4967, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732153551413 2024-11-21T01:45:54,095 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e425d310e254d2783ab4af61e135976, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732153552620 2024-11-21T01:45:54,103 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fd9dee8699da4e84c478aa5cfe68216#C#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-21T01:45:54,104 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/db41156d3e30425284a693e5b1980ce8 is 50, key is test_row_0/C:col10/1732153553301/Put/seqid=0 2024-11-21T01:45:54,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742316_1492 (size=16931) 2024-11-21T01:45:54,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/2abb11d59d9f4ddbb70eaaaa21a28db2 2024-11-21T01:45:54,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:54,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153614086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:54,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:54,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:54,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:54,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153614087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:54,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:54,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:54,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:54,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:54,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:54,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153614089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:54,110 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:54,110 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:54,110 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:54,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742317_1493 (size=12359) 2024-11-21T01:45:54,123 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/db41156d3e30425284a693e5b1980ce8 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/db41156d3e30425284a693e5b1980ce8 2024-11-21T01:45:54,130 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fd9dee8699da4e84c478aa5cfe68216/C of 7fd9dee8699da4e84c478aa5cfe68216 into db41156d3e30425284a693e5b1980ce8(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:54,130 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:45:54,130 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., storeName=7fd9dee8699da4e84c478aa5cfe68216/C, priority=13, startTime=1732153553970; duration=0sec 2024-11-21T01:45:54,131 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:54,131 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fd9dee8699da4e84c478aa5cfe68216:C 2024-11-21T01:45:54,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/5a01435af0e142b4b732b5e3183fb362 is 50, key is test_row_0/B:col10/1732153553377/Put/seqid=0 2024-11-21T01:45:54,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742318_1494 (size=12151) 2024-11-21T01:45:54,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T01:45:54,178 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:45:54,179 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:45:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-21T01:45:54,181 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:45:54,183 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:45:54,183 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:45:54,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T01:45:54,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153614213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:54,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153614213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:54,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:54,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153614213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:54,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:54,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:54,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:54,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:54,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:54,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:54,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:54,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T01:45:54,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-21T01:45:54,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:54,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:45:54,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:54,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:54,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:54,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:54,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153614423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:54,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:54,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:54,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:54,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153614423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:54,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:54,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:54,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:54,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:54,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:54,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153614423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:54,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:54,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:54,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:54,470 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/180751cadac140faaa1d2add6a1e33e5 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/180751cadac140faaa1d2add6a1e33e5 2024-11-21T01:45:54,484 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fd9dee8699da4e84c478aa5cfe68216/A of 7fd9dee8699da4e84c478aa5cfe68216 into 180751cadac140faaa1d2add6a1e33e5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:54,484 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:45:54,484 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., storeName=7fd9dee8699da4e84c478aa5cfe68216/A, priority=13, startTime=1732153553969; duration=0sec 2024-11-21T01:45:54,484 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:54,484 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fd9dee8699da4e84c478aa5cfe68216:A 2024-11-21T01:45:54,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-21T01:45:54,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:54,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:45:54,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:54,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:54,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T01:45:54,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/5a01435af0e142b4b732b5e3183fb362 2024-11-21T01:45:54,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/a185d36597a14e84af246c34467b3851 is 50, key is test_row_0/C:col10/1732153553377/Put/seqid=0 2024-11-21T01:45:54,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742319_1495 (size=12151) 2024-11-21T01:45:54,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/a185d36597a14e84af246c34467b3851 2024-11-21T01:45:54,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/2abb11d59d9f4ddbb70eaaaa21a28db2 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/2abb11d59d9f4ddbb70eaaaa21a28db2 2024-11-21T01:45:54,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/2abb11d59d9f4ddbb70eaaaa21a28db2, entries=250, sequenceid=153, filesize=16.5 K 2024-11-21T01:45:54,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/5a01435af0e142b4b732b5e3183fb362 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/5a01435af0e142b4b732b5e3183fb362 2024-11-21T01:45:54,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/5a01435af0e142b4b732b5e3183fb362, entries=150, sequenceid=153, filesize=11.9 K 2024-11-21T01:45:54,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/a185d36597a14e84af246c34467b3851 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/a185d36597a14e84af246c34467b3851 2024-11-21T01:45:54,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/a185d36597a14e84af246c34467b3851, entries=150, sequenceid=153, filesize=11.9 K 2024-11-21T01:45:54,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 7fd9dee8699da4e84c478aa5cfe68216 in 582ms, sequenceid=153, compaction requested=false 2024-11-21T01:45:54,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:45:54,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-21T01:45:54,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:54,654 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 7fd9dee8699da4e84c478aa5cfe68216 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T01:45:54,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=A 2024-11-21T01:45:54,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:54,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=B 2024-11-21T01:45:54,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:54,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=C 2024-11-21T01:45:54,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:54,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/a816e8ed79b94bb49234d03531c17d20 is 50, key is test_row_0/A:col10/1732153554084/Put/seqid=0 2024-11-21T01:45:54,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742320_1496 (size=12151) 2024-11-21T01:45:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:45:54,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:45:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T01:45:54,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732153554784, max timestamp=1732153554840Num uniques 859; 2024-11-21T01:45:54,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T01:45:54,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732153554814, max timestamp=1732153554840Num uniques 859; 2024-11-21T01:45:54,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T01:45:54,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732153554814, max timestamp=1732153554840Num uniques 859; 2024-11-21T01:45:54,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T01:45:54,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153614899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:54,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:54,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:54,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:54,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153614910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:54,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:54,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:54,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:54,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153614915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:54,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:54,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:54,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:54,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153614915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:54,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153614922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:54,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:54,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:54,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:54,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:54,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:54,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:55,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153615025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:55,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153615033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:55,047 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:55,047 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:55,047 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:55,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:55,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:55,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:55,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153615043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:55,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:55,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:55,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:55,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153615043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:55,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:55,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:55,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:55,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153615044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:55,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:55,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:55,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:55,109 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/a816e8ed79b94bb49234d03531c17d20 2024-11-21T01:45:55,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/bf8ef3853a784e93847acd58c2929a28 is 50, key is test_row_0/B:col10/1732153554084/Put/seqid=0 2024-11-21T01:45:55,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742321_1497 (size=12151) 2024-11-21T01:45:55,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:55,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153615257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:55,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:55,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153615258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:55,265 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:55,265 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:55,265 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:55,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:55,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:55,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:55,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:55,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153615263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:55,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:55,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:55,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:55,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:55,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153615263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:55,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:55,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153615264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:55,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:55,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:55,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:55,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T01:45:55,550 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/bf8ef3853a784e93847acd58c2929a28 2024-11-21T01:45:55,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153615573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:55,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153615574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:55,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:55,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:55,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:55,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:55,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:55,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:55,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/3286fc15af1b48bb93f8039b47f522ae is 50, key is test_row_0/C:col10/1732153554084/Put/seqid=0 2024-11-21T01:45:55,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153615585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:55,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:55,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:55,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:55,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153615595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:55,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:55,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:55,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:55,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:55,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153615595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:55,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:55,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:55,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:55,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742322_1498 (size=12151) 2024-11-21T01:45:56,036 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/3286fc15af1b48bb93f8039b47f522ae 2024-11-21T01:45:56,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/a816e8ed79b94bb49234d03531c17d20 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/a816e8ed79b94bb49234d03531c17d20 2024-11-21T01:45:56,061 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/a816e8ed79b94bb49234d03531c17d20, entries=150, sequenceid=168, filesize=11.9 K 2024-11-21T01:45:56,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/bf8ef3853a784e93847acd58c2929a28 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/bf8ef3853a784e93847acd58c2929a28 2024-11-21T01:45:56,068 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/bf8ef3853a784e93847acd58c2929a28, entries=150, sequenceid=168, filesize=11.9 K 2024-11-21T01:45:56,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/3286fc15af1b48bb93f8039b47f522ae as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/3286fc15af1b48bb93f8039b47f522ae 2024-11-21T01:45:56,076 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/3286fc15af1b48bb93f8039b47f522ae, entries=150, sequenceid=168, filesize=11.9 K 2024-11-21T01:45:56,077 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=123.00 KB/125950 for 7fd9dee8699da4e84c478aa5cfe68216 in 1423ms, sequenceid=168, compaction requested=true 2024-11-21T01:45:56,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:45:56,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:56,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-21T01:45:56,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-21T01:45:56,090 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-21T01:45:56,090 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9000 sec 2024-11-21T01:45:56,092 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.9120 sec 2024-11-21T01:45:56,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fd9dee8699da4e84c478aa5cfe68216 3/3 column families, dataSize=129.71 KB heapSize=370.14 KB 2024-11-21T01:45:56,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=A 2024-11-21T01:45:56,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:45:56,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=B 2024-11-21T01:45:56,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:45:56,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=C 2024-11-21T01:45:56,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:45:56,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:45:56,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/f94ae44c6beb458aa9680556755937ba is 50, key is test_row_0/A:col10/1732153554918/Put/seqid=0 2024-11-21T01:45:56,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742323_1499 (size=14541) 2024-11-21T01:45:56,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:56,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153616136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:56,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:56,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:56,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153616140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:56,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:56,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153616141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:56,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:56,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153616143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:56,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:56,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:56,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153616144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:56,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:56,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:56,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:56,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:56,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153616253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:56,261 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,261 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,261 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:56,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153616273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:56,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153616273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153616273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:56,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:56,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153616273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:56,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:56,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T01:45:56,343 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:45:56,348 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:45:56,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-21T01:45:56,351 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:45:56,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-21T01:45:56,352 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:45:56,352 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:45:56,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-21T01:45:56,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:56,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153616473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:56,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153616494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:56,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153616494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:56,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153616494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:56,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:56,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:56,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153616495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:56,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:56,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:56,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:56,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-21T01:45:56,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:56,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:45:56,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:56,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:56,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:56,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:56,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/f94ae44c6beb458aa9680556755937ba 2024-11-21T01:45:56,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/055487325a534ee5b14a317235591439 is 50, key is test_row_0/B:col10/1732153554918/Put/seqid=0 2024-11-21T01:45:56,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742324_1500 (size=12151) 2024-11-21T01:45:56,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-21T01:45:56,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:56,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:45:56,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:56,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:56,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:56,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-21T01:45:56,730 WARN [regionserver/b93ada835ef8:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 0 2024-11-21T01:45:56,730 DEBUG [regionserver/b93ada835ef8:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.0, tune throughput to 50.00 MB/second 2024-11-21T01:45:56,735 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/4682bcb408fe4ecb9d378a57773e8e05, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/a5b1ac8a547d403a90979cc9be63246b, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/0e2fbdc9682f44ce8aa57ed5ca020d31, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/978e9238355741b2adc373ced769e159, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/3f9a0a57cf784c139419509937ecbf8d, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/73633fdda07e450ca9a018bd898522ae, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/914b9ce601b7490a84f8ec1c065c9342, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/aac946e8917d4ae69fa1b4cfc7e5280b, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/45c675a3512f4fcea05619bc5c3c0396] to archive 2024-11-21T01:45:56,736 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T01:45:56,752 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/4682bcb408fe4ecb9d378a57773e8e05 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/4682bcb408fe4ecb9d378a57773e8e05 2024-11-21T01:45:56,760 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/a5b1ac8a547d403a90979cc9be63246b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/a5b1ac8a547d403a90979cc9be63246b 2024-11-21T01:45:56,764 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/0e2fbdc9682f44ce8aa57ed5ca020d31 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/0e2fbdc9682f44ce8aa57ed5ca020d31 2024-11-21T01:45:56,765 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/978e9238355741b2adc373ced769e159 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/978e9238355741b2adc373ced769e159 2024-11-21T01:45:56,766 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/3f9a0a57cf784c139419509937ecbf8d to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/3f9a0a57cf784c139419509937ecbf8d 2024-11-21T01:45:56,768 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/73633fdda07e450ca9a018bd898522ae to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/73633fdda07e450ca9a018bd898522ae 2024-11-21T01:45:56,769 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/914b9ce601b7490a84f8ec1c065c9342 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/914b9ce601b7490a84f8ec1c065c9342 2024-11-21T01:45:56,770 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/aac946e8917d4ae69fa1b4cfc7e5280b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/aac946e8917d4ae69fa1b4cfc7e5280b 2024-11-21T01:45:56,771 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/45c675a3512f4fcea05619bc5c3c0396 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/45c675a3512f4fcea05619bc5c3c0396 2024-11-21T01:45:56,774 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/274ee6cdee0f41d79def78c75cca2d99, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/300e254c1d854d359b3fb9d6ad0586bd, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/cb8e8c2bf78841aeb5ba833eabe1c9dc, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/5c956128cf96498382d3e1dac0cc8a64, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/aa132045d940464fb6a1accaee84625a, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/7ca412052eeb470bbecc09f447937138, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/75ea9367ec21478e946fc4a028123385, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/ea693fdba7904eec86e92512c16d02c8, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/4e1131e830714e11bae9a6380925af6a] to archive 2024-11-21T01:45:56,775 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T01:45:56,776 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/274ee6cdee0f41d79def78c75cca2d99 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/274ee6cdee0f41d79def78c75cca2d99 2024-11-21T01:45:56,782 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/300e254c1d854d359b3fb9d6ad0586bd to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/300e254c1d854d359b3fb9d6ad0586bd 2024-11-21T01:45:56,783 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/cb8e8c2bf78841aeb5ba833eabe1c9dc to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/cb8e8c2bf78841aeb5ba833eabe1c9dc 2024-11-21T01:45:56,784 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/5c956128cf96498382d3e1dac0cc8a64 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/5c956128cf96498382d3e1dac0cc8a64 2024-11-21T01:45:56,785 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/aa132045d940464fb6a1accaee84625a to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/aa132045d940464fb6a1accaee84625a 2024-11-21T01:45:56,786 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/7ca412052eeb470bbecc09f447937138 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/7ca412052eeb470bbecc09f447937138 2024-11-21T01:45:56,787 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/75ea9367ec21478e946fc4a028123385 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/75ea9367ec21478e946fc4a028123385 2024-11-21T01:45:56,787 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/ea693fdba7904eec86e92512c16d02c8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/ea693fdba7904eec86e92512c16d02c8 2024-11-21T01:45:56,788 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/4e1131e830714e11bae9a6380925af6a to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/4e1131e830714e11bae9a6380925af6a 2024-11-21T01:45:56,790 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/ae2427a4441140f7ada3260d01368015, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/395b3333fbeb402bb1211e1e4faf5c5b, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/c6abfbfcfa5147f9871956523f58cc26, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/974d1cdfe19d49a39113a705b80de60e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/9b97dcc6b60745c18d048209821e28eb, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/cc6a3c30149f425c960e3cadaf80d867, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/4824db2ed3a34d1aae63077d03ee41c9, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/be18d998e2b441c29912257acacd4967, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/0e425d310e254d2783ab4af61e135976] to archive 2024-11-21T01:45:56,790 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T01:45:56,791 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/ae2427a4441140f7ada3260d01368015 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/ae2427a4441140f7ada3260d01368015 2024-11-21T01:45:56,792 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/395b3333fbeb402bb1211e1e4faf5c5b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/395b3333fbeb402bb1211e1e4faf5c5b 2024-11-21T01:45:56,793 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/c6abfbfcfa5147f9871956523f58cc26 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/c6abfbfcfa5147f9871956523f58cc26 2024-11-21T01:45:56,794 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/974d1cdfe19d49a39113a705b80de60e to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/974d1cdfe19d49a39113a705b80de60e 2024-11-21T01:45:56,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:56,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153616783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:56,795 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,795 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,795 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:56,797 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/9b97dcc6b60745c18d048209821e28eb to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/9b97dcc6b60745c18d048209821e28eb 2024-11-21T01:45:56,798 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/cc6a3c30149f425c960e3cadaf80d867 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/cc6a3c30149f425c960e3cadaf80d867 2024-11-21T01:45:56,799 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/4824db2ed3a34d1aae63077d03ee41c9 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/4824db2ed3a34d1aae63077d03ee41c9 2024-11-21T01:45:56,800 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/be18d998e2b441c29912257acacd4967 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/be18d998e2b441c29912257acacd4967 2024-11-21T01:45:56,801 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/0e425d310e254d2783ab4af61e135976 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/0e425d310e254d2783ab4af61e135976 2024-11-21T01:45:56,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-21T01:45:56,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:56,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:45:56,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:56,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:56,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:56,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:56,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:56,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153616802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:56,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153616802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:56,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153616803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:56,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153616803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:56,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-21T01:45:56,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/055487325a534ee5b14a317235591439 2024-11-21T01:45:56,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:56,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:45:56,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:56,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:56,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:56,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:56,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/9d9838e46aff4f5084e869eff16108bb is 50, key is test_row_0/C:col10/1732153554918/Put/seqid=0 2024-11-21T01:45:56,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742325_1501 (size=12151) 2024-11-21T01:45:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-21T01:45:56,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/9d9838e46aff4f5084e869eff16108bb 2024-11-21T01:45:56,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/f94ae44c6beb458aa9680556755937ba as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/f94ae44c6beb458aa9680556755937ba 2024-11-21T01:45:57,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/f94ae44c6beb458aa9680556755937ba, entries=200, sequenceid=204, filesize=14.2 K 2024-11-21T01:45:57,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/055487325a534ee5b14a317235591439 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/055487325a534ee5b14a317235591439 2024-11-21T01:45:57,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/055487325a534ee5b14a317235591439, entries=150, sequenceid=204, filesize=11.9 K 2024-11-21T01:45:57,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/9d9838e46aff4f5084e869eff16108bb as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/9d9838e46aff4f5084e869eff16108bb 2024-11-21T01:45:57,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/9d9838e46aff4f5084e869eff16108bb, entries=150, sequenceid=204, filesize=11.9 K 2024-11-21T01:45:57,024 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~370.09 KB/378976, currentSize=53.67 KB/54960 for 7fd9dee8699da4e84c478aa5cfe68216 in 908ms, sequenceid=204, compaction requested=true 2024-11-21T01:45:57,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:45:57,024 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:45:57,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fd9dee8699da4e84c478aa5cfe68216:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:45:57,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:57,024 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:45:57,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fd9dee8699da4e84c478aa5cfe68216:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:45:57,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:57,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fd9dee8699da4e84c478aa5cfe68216:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:45:57,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:57,028 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55982 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:45:57,028 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48812 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:45:57,028 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 7fd9dee8699da4e84c478aa5cfe68216/A is initiating minor compaction (all files) 2024-11-21T01:45:57,028 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 7fd9dee8699da4e84c478aa5cfe68216/B is initiating minor compaction (all files) 2024-11-21T01:45:57,028 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fd9dee8699da4e84c478aa5cfe68216/A in TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:57,028 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fd9dee8699da4e84c478aa5cfe68216/B in TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:57,028 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/180751cadac140faaa1d2add6a1e33e5, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/2abb11d59d9f4ddbb70eaaaa21a28db2, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/a816e8ed79b94bb49234d03531c17d20, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/f94ae44c6beb458aa9680556755937ba] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp, totalSize=54.7 K 2024-11-21T01:45:57,028 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/9e004e4e620a43fba94885919e1ae245, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/5a01435af0e142b4b732b5e3183fb362, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/bf8ef3853a784e93847acd58c2929a28, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/055487325a534ee5b14a317235591439] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp, totalSize=47.7 K 2024-11-21T01:45:57,029 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 180751cadac140faaa1d2add6a1e33e5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732153552620 2024-11-21T01:45:57,029 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e004e4e620a43fba94885919e1ae245, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732153552620 2024-11-21T01:45:57,029 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2abb11d59d9f4ddbb70eaaaa21a28db2, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732153553352 2024-11-21T01:45:57,029 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a01435af0e142b4b732b5e3183fb362, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732153553377 2024-11-21T01:45:57,030 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting bf8ef3853a784e93847acd58c2929a28, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732153554069 2024-11-21T01:45:57,030 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting a816e8ed79b94bb49234d03531c17d20, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732153554069 2024-11-21T01:45:57,031 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting f94ae44c6beb458aa9680556755937ba, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732153554904 2024-11-21T01:45:57,031 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 055487325a534ee5b14a317235591439, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732153554910 2024-11-21T01:45:57,047 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fd9dee8699da4e84c478aa5cfe68216#A#compaction#427 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:45:57,048 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/c418c12c935a4c7483f3e5adf1882e2c is 50, key is test_row_0/A:col10/1732153554918/Put/seqid=0 2024-11-21T01:45:57,055 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fd9dee8699da4e84c478aa5cfe68216#B#compaction#426 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:45:57,056 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/77edf1cf02d2465d972befd3db56fb7f is 50, key is test_row_0/B:col10/1732153554918/Put/seqid=0 2024-11-21T01:45:57,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742326_1502 (size=12289) 2024-11-21T01:45:57,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742327_1503 (size=12289) 2024-11-21T01:45:57,069 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/c418c12c935a4c7483f3e5adf1882e2c as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/c418c12c935a4c7483f3e5adf1882e2c 2024-11-21T01:45:57,078 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7fd9dee8699da4e84c478aa5cfe68216/A of 7fd9dee8699da4e84c478aa5cfe68216 into c418c12c935a4c7483f3e5adf1882e2c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:57,078 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:45:57,078 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., storeName=7fd9dee8699da4e84c478aa5cfe68216/A, priority=12, startTime=1732153557024; duration=0sec 2024-11-21T01:45:57,078 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:57,078 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fd9dee8699da4e84c478aa5cfe68216:A 2024-11-21T01:45:57,078 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:45:57,079 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48812 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:45:57,079 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 7fd9dee8699da4e84c478aa5cfe68216/C is initiating minor compaction (all files) 2024-11-21T01:45:57,079 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fd9dee8699da4e84c478aa5cfe68216/C in TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:57,079 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/db41156d3e30425284a693e5b1980ce8, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/a185d36597a14e84af246c34467b3851, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/3286fc15af1b48bb93f8039b47f522ae, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/9d9838e46aff4f5084e869eff16108bb] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp, totalSize=47.7 K 2024-11-21T01:45:57,079 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting db41156d3e30425284a693e5b1980ce8, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732153552620 2024-11-21T01:45:57,080 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting a185d36597a14e84af246c34467b3851, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732153553377 2024-11-21T01:45:57,081 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3286fc15af1b48bb93f8039b47f522ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732153554069 2024-11-21T01:45:57,081 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d9838e46aff4f5084e869eff16108bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732153554910 2024-11-21T01:45:57,093 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fd9dee8699da4e84c478aa5cfe68216#C#compaction#428 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:45:57,093 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/ead5230fb3394dc4bf90ed38080e9a3e is 50, key is test_row_0/C:col10/1732153554918/Put/seqid=0 2024-11-21T01:45:57,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742328_1504 (size=12289) 2024-11-21T01:45:57,118 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/ead5230fb3394dc4bf90ed38080e9a3e as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/ead5230fb3394dc4bf90ed38080e9a3e 2024-11-21T01:45:57,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-21T01:45:57,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:57,125 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 7fd9dee8699da4e84c478aa5cfe68216 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-21T01:45:57,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=A 2024-11-21T01:45:57,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:57,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=B 2024-11-21T01:45:57,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:57,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=C 2024-11-21T01:45:57,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:57,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/2d10e65277794c00bc23d5790b5982d4 is 50, key is test_row_0/A:col10/1732153556142/Put/seqid=0 2024-11-21T01:45:57,143 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7fd9dee8699da4e84c478aa5cfe68216/C of 7fd9dee8699da4e84c478aa5cfe68216 into ead5230fb3394dc4bf90ed38080e9a3e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:57,143 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:45:57,143 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., storeName=7fd9dee8699da4e84c478aa5cfe68216/C, priority=12, startTime=1732153557025; duration=0sec 2024-11-21T01:45:57,143 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:57,143 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fd9dee8699da4e84c478aa5cfe68216:C 2024-11-21T01:45:57,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742329_1505 (size=12151) 2024-11-21T01:45:57,181 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/2d10e65277794c00bc23d5790b5982d4 2024-11-21T01:45:57,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/32700fbc27fd4289bbeaa35168e427e8 is 50, key is test_row_0/B:col10/1732153556142/Put/seqid=0 2024-11-21T01:45:57,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742330_1506 (size=12151) 2024-11-21T01:45:57,195 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/32700fbc27fd4289bbeaa35168e427e8 2024-11-21T01:45:57,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/7a8c14b0756149d5825e237e011da242 is 50, key is test_row_0/C:col10/1732153556142/Put/seqid=0 2024-11-21T01:45:57,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742331_1507 (size=12151) 2024-11-21T01:45:57,211 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/7a8c14b0756149d5825e237e011da242 2024-11-21T01:45:57,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/2d10e65277794c00bc23d5790b5982d4 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/2d10e65277794c00bc23d5790b5982d4 2024-11-21T01:45:57,221 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/2d10e65277794c00bc23d5790b5982d4, entries=150, sequenceid=216, filesize=11.9 K 2024-11-21T01:45:57,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/32700fbc27fd4289bbeaa35168e427e8 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/32700fbc27fd4289bbeaa35168e427e8 2024-11-21T01:45:57,231 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/32700fbc27fd4289bbeaa35168e427e8, entries=150, sequenceid=216, filesize=11.9 K 2024-11-21T01:45:57,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/7a8c14b0756149d5825e237e011da242 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/7a8c14b0756149d5825e237e011da242 2024-11-21T01:45:57,236 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/7a8c14b0756149d5825e237e011da242, entries=150, sequenceid=216, filesize=11.9 K 2024-11-21T01:45:57,237 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for 7fd9dee8699da4e84c478aa5cfe68216 in 112ms, sequenceid=216, compaction requested=false 2024-11-21T01:45:57,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:45:57,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:57,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-21T01:45:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-21T01:45:57,241 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-21T01:45:57,241 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 887 msec 2024-11-21T01:45:57,243 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 894 msec 2024-11-21T01:45:57,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fd9dee8699da4e84c478aa5cfe68216 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-21T01:45:57,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=A 2024-11-21T01:45:57,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:57,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=B 2024-11-21T01:45:57,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:57,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=C 2024-11-21T01:45:57,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:45:57,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/15c7c787f06c48f4883e190f459ddf65 is 50, key is test_row_0/A:col10/1732153557327/Put/seqid=0 2024-11-21T01:45:57,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742332_1508 (size=12151) 2024-11-21T01:45:57,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/15c7c787f06c48f4883e190f459ddf65 2024-11-21T01:45:57,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/7b5a988fc6f24c148d99fe63cdcfc3cf is 50, key is test_row_0/B:col10/1732153557327/Put/seqid=0 2024-11-21T01:45:57,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742333_1509 (size=12151) 2024-11-21T01:45:57,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:57,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153617391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:57,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732153557374, max timestamp=1732153557403Num uniques 859; 2024-11-21T01:45:57,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-21T01:45:57,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:57,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:57,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:57,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732153557374, max timestamp=1732153557403Num uniques 859; 2024-11-21T01:45:57,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-21T01:45:57,414 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732153557374, max timestamp=1732153557403Num uniques 859; 2024-11-21T01:45:57,414 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-21T01:45:57,470 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/77edf1cf02d2465d972befd3db56fb7f as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/77edf1cf02d2465d972befd3db56fb7f 2024-11-21T01:45:57,475 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7fd9dee8699da4e84c478aa5cfe68216/B of 7fd9dee8699da4e84c478aa5cfe68216 into 77edf1cf02d2465d972befd3db56fb7f(size=12.0 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:57,475 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:45:57,475 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., storeName=7fd9dee8699da4e84c478aa5cfe68216/B, priority=12, startTime=1732153557024; duration=0sec 2024-11-21T01:45:57,475 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:57,475 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fd9dee8699da4e84c478aa5cfe68216:B 2024-11-21T01:45:57,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153617475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:57,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:57,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:57,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-21T01:45:57,496 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:45:57,497 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:45:57,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153617478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:57,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153617479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:57,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153617476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:57,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-21T01:45:57,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:57,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:57,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:57,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:57,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:57,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:57,505 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:45:57,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T01:45:57,508 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:45:57,508 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:45:57,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153617513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:57,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:57,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153617593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:57,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:57,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T01:45:57,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:57,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:57,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:57,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153617602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:57,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:57,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:57,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:57,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:57,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153617613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:57,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:57,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:57,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:57,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:57,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153617613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:57,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:57,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:57,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:57,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-21T01:45:57,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:57,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:45:57,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:57,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:57,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:57,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:57,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:57,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153617733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:57,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:57,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:57,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:57,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/7b5a988fc6f24c148d99fe63cdcfc3cf 2024-11-21T01:45:57,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/ff286b1a867e436f8cfd927b38a17f89 is 50, key is test_row_0/C:col10/1732153557327/Put/seqid=0 2024-11-21T01:45:57,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742334_1510 (size=12151) 2024-11-21T01:45:57,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-21T01:45:57,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:57,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:45:57,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:57,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:57,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:57,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:57,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T01:45:57,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:57,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153617823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:57,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:57,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:57,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:57,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153617824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:57,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:57,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:57,836 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:57,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153617824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:57,836 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:57,836 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:57,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:57,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:57,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:57,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:57,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153617824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:57,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:57,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:57,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:57,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-21T01:45:57,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:57,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:45:57,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:57,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:57,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:57,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:58,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:58,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153618063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:58,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-21T01:45:58,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:58,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:45:58,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:58,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:58,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:58,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:58,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T01:45:58,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:58,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153618142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:58,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:58,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:58,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:58,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153618143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:58,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:58,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:58,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:58,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:58,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153618143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:58,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:58,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:58,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153618143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:58,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/ff286b1a867e436f8cfd927b38a17f89 2024-11-21T01:45:58,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/15c7c787f06c48f4883e190f459ddf65 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/15c7c787f06c48f4883e190f459ddf65 2024-11-21T01:45:58,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/15c7c787f06c48f4883e190f459ddf65, entries=150, sequenceid=228, filesize=11.9 K 2024-11-21T01:45:58,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/7b5a988fc6f24c148d99fe63cdcfc3cf as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/7b5a988fc6f24c148d99fe63cdcfc3cf 2024-11-21T01:45:58,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/7b5a988fc6f24c148d99fe63cdcfc3cf, entries=150, sequenceid=228, filesize=11.9 K 2024-11-21T01:45:58,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/ff286b1a867e436f8cfd927b38a17f89 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/ff286b1a867e436f8cfd927b38a17f89 2024-11-21T01:45:58,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/ff286b1a867e436f8cfd927b38a17f89, entries=150, sequenceid=228, filesize=11.9 K 2024-11-21T01:45:58,222 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 7fd9dee8699da4e84c478aa5cfe68216 in 887ms, sequenceid=228, compaction requested=true 2024-11-21T01:45:58,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:45:58,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fd9dee8699da4e84c478aa5cfe68216:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:45:58,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:58,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fd9dee8699da4e84c478aa5cfe68216:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:45:58,222 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:58,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:58,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fd9dee8699da4e84c478aa5cfe68216:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:45:58,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:58,222 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:58,223 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36591 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:58,223 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36591 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:58,223 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 7fd9dee8699da4e84c478aa5cfe68216/A is initiating minor compaction (all files) 2024-11-21T01:45:58,223 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 7fd9dee8699da4e84c478aa5cfe68216/B is initiating minor compaction (all files) 2024-11-21T01:45:58,223 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fd9dee8699da4e84c478aa5cfe68216/B in TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:58,223 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fd9dee8699da4e84c478aa5cfe68216/A in TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:58,223 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/77edf1cf02d2465d972befd3db56fb7f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/32700fbc27fd4289bbeaa35168e427e8, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/7b5a988fc6f24c148d99fe63cdcfc3cf] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp, totalSize=35.7 K 2024-11-21T01:45:58,223 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/c418c12c935a4c7483f3e5adf1882e2c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/2d10e65277794c00bc23d5790b5982d4, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/15c7c787f06c48f4883e190f459ddf65] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp, totalSize=35.7 K 2024-11-21T01:45:58,225 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting c418c12c935a4c7483f3e5adf1882e2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732153554910 2024-11-21T01:45:58,226 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 77edf1cf02d2465d972befd3db56fb7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732153554910 2024-11-21T01:45:58,226 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d10e65277794c00bc23d5790b5982d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732153556135 2024-11-21T01:45:58,226 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 32700fbc27fd4289bbeaa35168e427e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732153556135 2024-11-21T01:45:58,227 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15c7c787f06c48f4883e190f459ddf65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732153557327 2024-11-21T01:45:58,227 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b5a988fc6f24c148d99fe63cdcfc3cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732153557327 2024-11-21T01:45:58,237 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fd9dee8699da4e84c478aa5cfe68216#A#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:45:58,237 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fd9dee8699da4e84c478aa5cfe68216#B#compaction#436 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:45:58,237 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/d91b3177e3de49a691e3aee194c6a0f5 is 50, key is test_row_0/A:col10/1732153557327/Put/seqid=0 2024-11-21T01:45:58,238 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/3d90bf854509439db82b1d8c678880ab is 50, key is test_row_0/B:col10/1732153557327/Put/seqid=0 2024-11-21T01:45:58,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742335_1511 (size=12391) 2024-11-21T01:45:58,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742336_1512 (size=12391) 2024-11-21T01:45:58,265 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/3d90bf854509439db82b1d8c678880ab as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/3d90bf854509439db82b1d8c678880ab 2024-11-21T01:45:58,273 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fd9dee8699da4e84c478aa5cfe68216/B of 7fd9dee8699da4e84c478aa5cfe68216 into 3d90bf854509439db82b1d8c678880ab(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:58,273 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:45:58,273 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., storeName=7fd9dee8699da4e84c478aa5cfe68216/B, priority=13, startTime=1732153558222; duration=0sec 2024-11-21T01:45:58,273 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:45:58,273 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fd9dee8699da4e84c478aa5cfe68216:B 2024-11-21T01:45:58,273 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:45:58,274 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/d91b3177e3de49a691e3aee194c6a0f5 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/d91b3177e3de49a691e3aee194c6a0f5 2024-11-21T01:45:58,274 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36591 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:45:58,274 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 7fd9dee8699da4e84c478aa5cfe68216/C is initiating minor compaction (all files) 2024-11-21T01:45:58,274 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fd9dee8699da4e84c478aa5cfe68216/C in TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:58,274 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/ead5230fb3394dc4bf90ed38080e9a3e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/7a8c14b0756149d5825e237e011da242, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/ff286b1a867e436f8cfd927b38a17f89] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp, totalSize=35.7 K 2024-11-21T01:45:58,275 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting ead5230fb3394dc4bf90ed38080e9a3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732153554910 2024-11-21T01:45:58,277 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a8c14b0756149d5825e237e011da242, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732153556135 2024-11-21T01:45:58,277 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting ff286b1a867e436f8cfd927b38a17f89, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732153557327 2024-11-21T01:45:58,280 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fd9dee8699da4e84c478aa5cfe68216/A of 7fd9dee8699da4e84c478aa5cfe68216 into d91b3177e3de49a691e3aee194c6a0f5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:58,280 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:45:58,280 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., storeName=7fd9dee8699da4e84c478aa5cfe68216/A, priority=13, startTime=1732153558222; duration=0sec 2024-11-21T01:45:58,280 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:58,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-21T01:45:58,280 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fd9dee8699da4e84c478aa5cfe68216:A 2024-11-21T01:45:58,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:58,281 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 7fd9dee8699da4e84c478aa5cfe68216 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-21T01:45:58,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=A 2024-11-21T01:45:58,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:45:58,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=B 2024-11-21T01:45:58,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:45:58,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=C 2024-11-21T01:45:58,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:45:58,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/2714c72c47774ae2a59e5aa6a4e12849 is 50, key is test_row_0/A:col10/1732153557476/Put/seqid=0 2024-11-21T01:45:58,290 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fd9dee8699da4e84c478aa5cfe68216#C#compaction#438 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:45:58,290 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/643f7d8858684cf9b299510d2403d34c is 50, key is test_row_0/C:col10/1732153557327/Put/seqid=0 2024-11-21T01:45:58,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742337_1513 (size=12301) 2024-11-21T01:45:58,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742338_1514 (size=12391) 2024-11-21T01:45:58,297 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/643f7d8858684cf9b299510d2403d34c as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/643f7d8858684cf9b299510d2403d34c 2024-11-21T01:45:58,304 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fd9dee8699da4e84c478aa5cfe68216/C of 7fd9dee8699da4e84c478aa5cfe68216 into 643f7d8858684cf9b299510d2403d34c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:45:58,304 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:45:58,304 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., storeName=7fd9dee8699da4e84c478aa5cfe68216/C, priority=13, startTime=1732153558222; duration=0sec 2024-11-21T01:45:58,304 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:45:58,304 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fd9dee8699da4e84c478aa5cfe68216:C 2024-11-21T01:45:58,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:45:58,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:45:58,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T01:45:58,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:58,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153618653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:58,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:58,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153618657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:58,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:58,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153618663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:58,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:58,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153618663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:58,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:58,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153618664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:58,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:58,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:58,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:58,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:58,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:58,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:58,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:58,693 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/2714c72c47774ae2a59e5aa6a4e12849 2024-11-21T01:45:58,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/6f3feb3c8bea4daa94a90bc5b4217a0f is 50, key is test_row_0/B:col10/1732153557476/Put/seqid=0 2024-11-21T01:45:58,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742339_1515 (size=12301) 2024-11-21T01:45:58,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:58,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153618775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:58,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:58,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:58,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:58,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153618792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:58,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:58,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:58,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153618793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:58,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:58,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:58,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:58,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:59,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:59,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153618993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:59,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:59,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:59,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:59,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:59,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153619012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:59,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:59,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153619013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:59,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:59,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:59,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:59,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:59,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:59,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:59,110 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/6f3feb3c8bea4daa94a90bc5b4217a0f 2024-11-21T01:45:59,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/69ac80d25e8c49d78a44fc67b6a6cc8f is 50, key is test_row_0/C:col10/1732153557476/Put/seqid=0 2024-11-21T01:45:59,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742340_1516 (size=12301) 2024-11-21T01:45:59,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:59,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153619313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:59,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:59,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:59,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:59,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:59,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153619333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:59,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:59,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:59,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:59,344 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:59,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153619333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:59,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:59,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:59,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:59,524 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/69ac80d25e8c49d78a44fc67b6a6cc8f 2024-11-21T01:45:59,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/2714c72c47774ae2a59e5aa6a4e12849 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/2714c72c47774ae2a59e5aa6a4e12849 2024-11-21T01:45:59,533 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/2714c72c47774ae2a59e5aa6a4e12849, entries=150, sequenceid=265, filesize=12.0 K 2024-11-21T01:45:59,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/6f3feb3c8bea4daa94a90bc5b4217a0f as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/6f3feb3c8bea4daa94a90bc5b4217a0f 2024-11-21T01:45:59,538 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/6f3feb3c8bea4daa94a90bc5b4217a0f, entries=150, sequenceid=265, filesize=12.0 K 2024-11-21T01:45:59,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/69ac80d25e8c49d78a44fc67b6a6cc8f as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/69ac80d25e8c49d78a44fc67b6a6cc8f 2024-11-21T01:45:59,542 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/69ac80d25e8c49d78a44fc67b6a6cc8f, entries=150, sequenceid=265, filesize=12.0 K 2024-11-21T01:45:59,543 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for 7fd9dee8699da4e84c478aa5cfe68216 in 1262ms, sequenceid=265, compaction requested=false 2024-11-21T01:45:59,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:45:59,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:59,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-21T01:45:59,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-21T01:45:59,548 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-21T01:45:59,548 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0380 sec 2024-11-21T01:45:59,550 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 2.0520 sec 2024-11-21T01:45:59,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T01:45:59,664 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:45:59,664 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:45:59,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-21T01:45:59,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-21T01:45:59,666 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:45:59,667 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:45:59,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:45:59,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:45:59,678 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fd9dee8699da4e84c478aa5cfe68216 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T01:45:59,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=A 2024-11-21T01:45:59,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:59,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=B 2024-11-21T01:45:59,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:59,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=C 2024-11-21T01:45:59,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:45:59,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/2f35986e024a475b824378fd77858c90 is 50, key is test_row_0/A:col10/1732153559677/Put/seqid=0 2024-11-21T01:45:59,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742341_1517 (size=14741) 2024-11-21T01:45:59,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-21T01:45:59,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:59,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153619795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:59,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:59,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:59,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:59,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-21T01:45:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:45:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:59,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:59,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:59,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:59,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153619810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:59,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:59,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:59,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:59,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:59,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153619832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:59,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:59,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:59,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:59,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153619853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:59,865 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:59,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:59,865 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:59,865 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153619853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:59,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:59,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:59,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:59,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:59,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153619912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:59,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:59,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:59,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:59,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:45:59,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153619933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:45:59,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:59,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:45:59,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:45:59,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-21T01:45:59,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:59,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:45:59,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:45:59,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:59,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:45:59,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-21T01:46:00,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/2f35986e024a475b824378fd77858c90 2024-11-21T01:46:00,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/d0352424bf2d440cb0790284159d2715 is 50, key is test_row_0/B:col10/1732153559677/Put/seqid=0 2024-11-21T01:46:00,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742342_1518 (size=12301) 2024-11-21T01:46:00,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/d0352424bf2d440cb0790284159d2715 2024-11-21T01:46:00,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/cbc3fadf8ab142a6bbde875acfe2034d is 50, key is test_row_0/C:col10/1732153559677/Put/seqid=0 2024-11-21T01:46:00,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742343_1519 (size=12301) 2024-11-21T01:46:00,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-21T01:46:00,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:00,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:46:00,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:00,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:00,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:00,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:00,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153620134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:00,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:00,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:00,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:00,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:00,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153620152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:00,166 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:00,166 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:00,166 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:00,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-21T01:46:00,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:00,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:46:00,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:00,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:00,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:00,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:00,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-21T01:46:00,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-21T01:46:00,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:00,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:46:00,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:00,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:00,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:00,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153620443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:00,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:00,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:00,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153620474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:00,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:00,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:00,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:00,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/cbc3fadf8ab142a6bbde875acfe2034d 2024-11-21T01:46:00,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/2f35986e024a475b824378fd77858c90 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/2f35986e024a475b824378fd77858c90 2024-11-21T01:46:00,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/2f35986e024a475b824378fd77858c90, entries=200, sequenceid=278, filesize=14.4 K 2024-11-21T01:46:00,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/d0352424bf2d440cb0790284159d2715 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/d0352424bf2d440cb0790284159d2715 2024-11-21T01:46:00,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/d0352424bf2d440cb0790284159d2715, entries=150, sequenceid=278, filesize=12.0 K 2024-11-21T01:46:00,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/cbc3fadf8ab142a6bbde875acfe2034d as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/cbc3fadf8ab142a6bbde875acfe2034d 2024-11-21T01:46:00,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-21T01:46:00,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/cbc3fadf8ab142a6bbde875acfe2034d, entries=150, sequenceid=278, filesize=12.0 K 2024-11-21T01:46:00,586 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 7fd9dee8699da4e84c478aa5cfe68216 in 908ms, sequenceid=278, compaction requested=true 2024-11-21T01:46:00,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:46:00,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fd9dee8699da4e84c478aa5cfe68216:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:46:00,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:46:00,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fd9dee8699da4e84c478aa5cfe68216:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:46:00,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T01:46:00,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fd9dee8699da4e84c478aa5cfe68216:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:46:00,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-21T01:46:00,587 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:46:00,587 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:46:00,589 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:46:00,589 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 7fd9dee8699da4e84c478aa5cfe68216/C is initiating minor compaction (all files) 2024-11-21T01:46:00,589 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fd9dee8699da4e84c478aa5cfe68216/C in TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:00,589 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/643f7d8858684cf9b299510d2403d34c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/69ac80d25e8c49d78a44fc67b6a6cc8f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/cbc3fadf8ab142a6bbde875acfe2034d] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp, totalSize=36.1 K 2024-11-21T01:46:00,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:00,589 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:46:00,589 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing 7fd9dee8699da4e84c478aa5cfe68216 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T01:46:00,589 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 7fd9dee8699da4e84c478aa5cfe68216/A is initiating minor compaction (all files) 2024-11-21T01:46:00,589 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fd9dee8699da4e84c478aa5cfe68216/A in TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:00,589 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/d91b3177e3de49a691e3aee194c6a0f5, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/2714c72c47774ae2a59e5aa6a4e12849, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/2f35986e024a475b824378fd77858c90] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp, totalSize=38.5 K 2024-11-21T01:46:00,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=A 2024-11-21T01:46:00,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:00,590 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 643f7d8858684cf9b299510d2403d34c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732153557327 2024-11-21T01:46:00,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=B 2024-11-21T01:46:00,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:00,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=C 2024-11-21T01:46:00,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:00,591 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting d91b3177e3de49a691e3aee194c6a0f5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732153557327 2024-11-21T01:46:00,591 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 69ac80d25e8c49d78a44fc67b6a6cc8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732153557474 2024-11-21T01:46:00,591 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2714c72c47774ae2a59e5aa6a4e12849, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732153557474 2024-11-21T01:46:00,592 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting cbc3fadf8ab142a6bbde875acfe2034d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732153558640 2024-11-21T01:46:00,592 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f35986e024a475b824378fd77858c90, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732153558640 2024-11-21T01:46:00,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/c4a5dce64bd14fa095b07e86451a2787 is 50, key is test_row_0/A:col10/1732153559781/Put/seqid=0 2024-11-21T01:46:00,622 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fd9dee8699da4e84c478aa5cfe68216#C#compaction#445 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:00,623 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/7bddd417a45444468f702fb31f34b1fb is 50, key is test_row_0/C:col10/1732153559677/Put/seqid=0 2024-11-21T01:46:00,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742344_1520 (size=12301) 2024-11-21T01:46:00,648 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fd9dee8699da4e84c478aa5cfe68216#A#compaction#446 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:00,649 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/547baa89ea4c43919e218ee449a892dc is 50, key is test_row_0/A:col10/1732153559677/Put/seqid=0 2024-11-21T01:46:00,649 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/c4a5dce64bd14fa095b07e86451a2787 2024-11-21T01:46:00,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742345_1521 (size=12643) 2024-11-21T01:46:00,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742346_1522 (size=12643) 2024-11-21T01:46:00,693 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/547baa89ea4c43919e218ee449a892dc as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/547baa89ea4c43919e218ee449a892dc 2024-11-21T01:46:00,697 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fd9dee8699da4e84c478aa5cfe68216/A of 7fd9dee8699da4e84c478aa5cfe68216 into 547baa89ea4c43919e218ee449a892dc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:00,697 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:46:00,697 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., storeName=7fd9dee8699da4e84c478aa5cfe68216/A, priority=13, startTime=1732153560587; duration=0sec 2024-11-21T01:46:00,697 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:46:00,697 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fd9dee8699da4e84c478aa5cfe68216:A 2024-11-21T01:46:00,698 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:46:00,698 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:46:00,698 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 7fd9dee8699da4e84c478aa5cfe68216/B is initiating minor compaction (all files) 2024-11-21T01:46:00,699 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fd9dee8699da4e84c478aa5cfe68216/B in TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:00,699 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/3d90bf854509439db82b1d8c678880ab, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/6f3feb3c8bea4daa94a90bc5b4217a0f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/d0352424bf2d440cb0790284159d2715] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp, totalSize=36.1 K 2024-11-21T01:46:00,699 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d90bf854509439db82b1d8c678880ab, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732153557327 2024-11-21T01:46:00,699 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f3feb3c8bea4daa94a90bc5b4217a0f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732153557474 2024-11-21T01:46:00,700 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0352424bf2d440cb0790284159d2715, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732153558640 2024-11-21T01:46:00,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/8fd61161156143d5b0de329a6ca6d99c is 50, key is test_row_0/B:col10/1732153559781/Put/seqid=0 2024-11-21T01:46:00,749 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fd9dee8699da4e84c478aa5cfe68216#B#compaction#448 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:00,750 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/f809396577464406a5c1ad30a9a43dad is 50, key is test_row_0/B:col10/1732153559677/Put/seqid=0 2024-11-21T01:46:00,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742347_1523 (size=12301) 2024-11-21T01:46:00,765 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/8fd61161156143d5b0de329a6ca6d99c 2024-11-21T01:46:00,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742348_1524 (size=12643) 2024-11-21T01:46:00,804 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/f809396577464406a5c1ad30a9a43dad as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/f809396577464406a5c1ad30a9a43dad 2024-11-21T01:46:00,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/e8cf65e1de9e4035a7cf7a98c3391cc0 is 50, key is test_row_0/C:col10/1732153559781/Put/seqid=0 2024-11-21T01:46:00,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-21T01:46:00,814 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fd9dee8699da4e84c478aa5cfe68216/B of 7fd9dee8699da4e84c478aa5cfe68216 into f809396577464406a5c1ad30a9a43dad(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:00,814 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:46:00,814 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., storeName=7fd9dee8699da4e84c478aa5cfe68216/B, priority=13, startTime=1732153560587; duration=0sec 2024-11-21T01:46:00,815 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:00,815 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fd9dee8699da4e84c478aa5cfe68216:B 2024-11-21T01:46:00,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742349_1525 (size=12301) 2024-11-21T01:46:00,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:46:00,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:46:00,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:00,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153620893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:00,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:00,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:00,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:00,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:00,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153620894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:00,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:00,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:00,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:00,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:00,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153620896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:00,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:00,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:00,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:00,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:00,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153620952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:00,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:00,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:00,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:00,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:00,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153620993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:00,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:00,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:00,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:01,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:01,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153621013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:01,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:01,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:01,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:01,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153621013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:01,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:01,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:01,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:01,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:01,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153621023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:01,076 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/7bddd417a45444468f702fb31f34b1fb as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/7bddd417a45444468f702fb31f34b1fb 2024-11-21T01:46:01,081 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fd9dee8699da4e84c478aa5cfe68216/C of 7fd9dee8699da4e84c478aa5cfe68216 into 7bddd417a45444468f702fb31f34b1fb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:01,081 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:46:01,081 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., storeName=7fd9dee8699da4e84c478aa5cfe68216/C, priority=13, startTime=1732153560587; duration=0sec 2024-11-21T01:46:01,081 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:01,081 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fd9dee8699da4e84c478aa5cfe68216:C 2024-11-21T01:46:01,222 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/e8cf65e1de9e4035a7cf7a98c3391cc0 2024-11-21T01:46:01,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/c4a5dce64bd14fa095b07e86451a2787 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/c4a5dce64bd14fa095b07e86451a2787 2024-11-21T01:46:01,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153621222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:01,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153621222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:01,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:01,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:01,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:01,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:01,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:01,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:01,231 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/c4a5dce64bd14fa095b07e86451a2787, entries=150, sequenceid=298, filesize=12.0 K 2024-11-21T01:46:01,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/8fd61161156143d5b0de329a6ca6d99c as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/8fd61161156143d5b0de329a6ca6d99c 2024-11-21T01:46:01,236 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/8fd61161156143d5b0de329a6ca6d99c, entries=150, sequenceid=298, filesize=12.0 K 2024-11-21T01:46:01,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/e8cf65e1de9e4035a7cf7a98c3391cc0 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/e8cf65e1de9e4035a7cf7a98c3391cc0 2024-11-21T01:46:01,242 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/e8cf65e1de9e4035a7cf7a98c3391cc0, entries=150, sequenceid=298, filesize=12.0 K 2024-11-21T01:46:01,243 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 7fd9dee8699da4e84c478aa5cfe68216 in 654ms, sequenceid=298, compaction requested=false 2024-11-21T01:46:01,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:46:01,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:01,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-21T01:46:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-21T01:46:01,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-21T01:46:01,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5780 sec 2024-11-21T01:46:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:46:01,251 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 1.5860 sec 2024-11-21T01:46:01,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fd9dee8699da4e84c478aa5cfe68216 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T01:46:01,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=A 2024-11-21T01:46:01,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:01,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=B 2024-11-21T01:46:01,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:01,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=C 2024-11-21T01:46:01,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:01,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/9fa45c2d245e4443a49de321358f0fa8 is 50, key is test_row_0/A:col10/1732153561246/Put/seqid=0 2024-11-21T01:46:01,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742350_1526 (size=14741) 2024-11-21T01:46:01,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/9fa45c2d245e4443a49de321358f0fa8 2024-11-21T01:46:01,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/3b7d81d77e9a42ea822c59633227f28a is 50, key is test_row_0/B:col10/1732153561246/Put/seqid=0 2024-11-21T01:46:01,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742351_1527 (size=12301) 2024-11-21T01:46:01,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153621379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:01,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:01,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:01,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:01,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:01,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153621503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:01,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:01,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:01,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:01,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153621533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:01,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:01,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:01,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:01,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153621533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:01,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:01,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:01,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:01,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/3b7d81d77e9a42ea822c59633227f28a 2024-11-21T01:46:01,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/566c73149bf54f0188ae9efb55bc2174 is 50, key is test_row_0/C:col10/1732153561246/Put/seqid=0 2024-11-21T01:46:01,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742352_1528 (size=12301) 2024-11-21T01:46:01,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/566c73149bf54f0188ae9efb55bc2174 2024-11-21T01:46:01,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/9fa45c2d245e4443a49de321358f0fa8 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/9fa45c2d245e4443a49de321358f0fa8 2024-11-21T01:46:01,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/9fa45c2d245e4443a49de321358f0fa8, entries=200, sequenceid=314, filesize=14.4 K 2024-11-21T01:46:01,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/3b7d81d77e9a42ea822c59633227f28a as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/3b7d81d77e9a42ea822c59633227f28a 2024-11-21T01:46:01,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/3b7d81d77e9a42ea822c59633227f28a, entries=150, sequenceid=314, filesize=12.0 K 2024-11-21T01:46:01,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:01,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/566c73149bf54f0188ae9efb55bc2174 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/566c73149bf54f0188ae9efb55bc2174 2024-11-21T01:46:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153621713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:01,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:01,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:01,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:01,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/566c73149bf54f0188ae9efb55bc2174, entries=150, sequenceid=314, filesize=12.0 K 2024-11-21T01:46:01,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 7fd9dee8699da4e84c478aa5cfe68216 in 472ms, sequenceid=314, compaction requested=true 2024-11-21T01:46:01,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:46:01,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fd9dee8699da4e84c478aa5cfe68216:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:46:01,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:01,723 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:46:01,723 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:46:01,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fd9dee8699da4e84c478aa5cfe68216:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:46:01,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:01,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fd9dee8699da4e84c478aa5cfe68216:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:46:01,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:46:01,724 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39685 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:46:01,725 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 7fd9dee8699da4e84c478aa5cfe68216/A is initiating minor compaction (all files) 2024-11-21T01:46:01,725 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fd9dee8699da4e84c478aa5cfe68216/A in TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:01,725 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/547baa89ea4c43919e218ee449a892dc, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/c4a5dce64bd14fa095b07e86451a2787, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/9fa45c2d245e4443a49de321358f0fa8] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp, totalSize=38.8 K 2024-11-21T01:46:01,725 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 547baa89ea4c43919e218ee449a892dc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732153558640 2024-11-21T01:46:01,725 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:46:01,725 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 7fd9dee8699da4e84c478aa5cfe68216/B is initiating minor compaction (all files) 2024-11-21T01:46:01,725 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fd9dee8699da4e84c478aa5cfe68216/B in TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:01,725 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/f809396577464406a5c1ad30a9a43dad, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/8fd61161156143d5b0de329a6ca6d99c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/3b7d81d77e9a42ea822c59633227f28a] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp, totalSize=36.4 K 2024-11-21T01:46:01,726 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting f809396577464406a5c1ad30a9a43dad, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732153558640 2024-11-21T01:46:01,726 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4a5dce64bd14fa095b07e86451a2787, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732153559781 2024-11-21T01:46:01,727 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fa45c2d245e4443a49de321358f0fa8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732153560868 2024-11-21T01:46:01,727 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fd61161156143d5b0de329a6ca6d99c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732153559781 2024-11-21T01:46:01,727 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b7d81d77e9a42ea822c59633227f28a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732153560868 2024-11-21T01:46:01,736 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fd9dee8699da4e84c478aa5cfe68216#B#compaction#453 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:01,736 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/5e1a8528b6ea45f78def8a653941ba7c is 50, key is test_row_0/B:col10/1732153561246/Put/seqid=0 2024-11-21T01:46:01,739 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fd9dee8699da4e84c478aa5cfe68216#A#compaction#454 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:01,740 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/6ab79374da00475593735c31fe3a7c02 is 50, key is test_row_0/A:col10/1732153561246/Put/seqid=0 2024-11-21T01:46:01,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742353_1529 (size=12745) 2024-11-21T01:46:01,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742354_1530 (size=12745) 2024-11-21T01:46:01,751 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/6ab79374da00475593735c31fe3a7c02 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/6ab79374da00475593735c31fe3a7c02 2024-11-21T01:46:01,753 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/5e1a8528b6ea45f78def8a653941ba7c as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/5e1a8528b6ea45f78def8a653941ba7c 2024-11-21T01:46:01,757 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fd9dee8699da4e84c478aa5cfe68216/A of 7fd9dee8699da4e84c478aa5cfe68216 into 6ab79374da00475593735c31fe3a7c02(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:01,757 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:46:01,757 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., storeName=7fd9dee8699da4e84c478aa5cfe68216/A, priority=13, startTime=1732153561723; duration=0sec 2024-11-21T01:46:01,758 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:46:01,758 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fd9dee8699da4e84c478aa5cfe68216:A 2024-11-21T01:46:01,758 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:46:01,759 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:46:01,759 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 7fd9dee8699da4e84c478aa5cfe68216/C is initiating minor compaction (all files) 2024-11-21T01:46:01,759 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fd9dee8699da4e84c478aa5cfe68216/C in TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:01,759 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/7bddd417a45444468f702fb31f34b1fb, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/e8cf65e1de9e4035a7cf7a98c3391cc0, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/566c73149bf54f0188ae9efb55bc2174] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp, totalSize=36.4 K 2024-11-21T01:46:01,759 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fd9dee8699da4e84c478aa5cfe68216/B of 7fd9dee8699da4e84c478aa5cfe68216 into 5e1a8528b6ea45f78def8a653941ba7c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:01,759 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:46:01,759 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., storeName=7fd9dee8699da4e84c478aa5cfe68216/B, priority=13, startTime=1732153561723; duration=0sec 2024-11-21T01:46:01,760 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:01,760 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fd9dee8699da4e84c478aa5cfe68216:B 2024-11-21T01:46:01,760 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bddd417a45444468f702fb31f34b1fb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732153558640 2024-11-21T01:46:01,760 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8cf65e1de9e4035a7cf7a98c3391cc0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732153559781 2024-11-21T01:46:01,760 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 566c73149bf54f0188ae9efb55bc2174, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732153560868 2024-11-21T01:46:01,770 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fd9dee8699da4e84c478aa5cfe68216#C#compaction#455 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:01,770 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/3df195f162444d55a318151eee8e93f8 is 50, key is test_row_0/C:col10/1732153561246/Put/seqid=0 2024-11-21T01:46:01,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742355_1531 (size=12745) 2024-11-21T01:46:01,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-21T01:46:01,813 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:46:01,814 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:46:01,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees 2024-11-21T01:46:01,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-21T01:46:01,816 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:46:01,816 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:46:01,816 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:46:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-21T01:46:01,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-21T01:46:01,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:01,969 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2902): Flushing 7fd9dee8699da4e84c478aa5cfe68216 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T01:46:01,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=A 2024-11-21T01:46:01,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:01,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=B 2024-11-21T01:46:01,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:01,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=C 2024-11-21T01:46:01,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:01,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:46:01,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:46:01,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/f2b7edfcda3b40a6afc2db4222599d47 is 50, key is test_row_0/A:col10/1732153561378/Put/seqid=0 2024-11-21T01:46:01,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742356_1532 (size=12301) 2024-11-21T01:46:02,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:02,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153622018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:02,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:02,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:02,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:02,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:02,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153622022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:02,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:02,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:02,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:02,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:02,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153622027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:02,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:02,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:02,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:02,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:02,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153622043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:02,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:02,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:02,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:02,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:02,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153622053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-21T01:46:02,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:02,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153622133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:02,140 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:02,140 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:02,140 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:02,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:02,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153622153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:02,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:02,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:02,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:02,185 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/3df195f162444d55a318151eee8e93f8 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/3df195f162444d55a318151eee8e93f8 2024-11-21T01:46:02,191 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fd9dee8699da4e84c478aa5cfe68216/C of 7fd9dee8699da4e84c478aa5cfe68216 into 3df195f162444d55a318151eee8e93f8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:02,191 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:46:02,191 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., storeName=7fd9dee8699da4e84c478aa5cfe68216/C, priority=13, startTime=1732153561724; duration=0sec 2024-11-21T01:46:02,191 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:02,191 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fd9dee8699da4e84c478aa5cfe68216:C 2024-11-21T01:46:02,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:02,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153622353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:02,361 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:02,361 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:02,361 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:02,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:02,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153622363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:02,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:02,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:02,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:02,384 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/f2b7edfcda3b40a6afc2db4222599d47 2024-11-21T01:46:02,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/82323a83e51f44afa07de57938ee4978 is 50, key is test_row_0/B:col10/1732153561378/Put/seqid=0 2024-11-21T01:46:02,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742357_1533 (size=12301) 2024-11-21T01:46:02,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-21T01:46:02,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:02,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153622542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:02,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:02,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:02,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:02,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:02,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153622672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:02,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:02,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:02,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153622673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:02,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:02,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:02,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:02,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:02,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:02,811 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/82323a83e51f44afa07de57938ee4978 2024-11-21T01:46:02,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/7380c3c45e6c46fda87272ca0bb6c7a3 is 50, key is test_row_0/C:col10/1732153561378/Put/seqid=0 2024-11-21T01:46:02,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742358_1534 (size=12301) 2024-11-21T01:46:02,828 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/7380c3c45e6c46fda87272ca0bb6c7a3 2024-11-21T01:46:02,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/f2b7edfcda3b40a6afc2db4222599d47 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/f2b7edfcda3b40a6afc2db4222599d47 2024-11-21T01:46:02,836 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/f2b7edfcda3b40a6afc2db4222599d47, entries=150, sequenceid=335, filesize=12.0 K 2024-11-21T01:46:02,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/82323a83e51f44afa07de57938ee4978 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/82323a83e51f44afa07de57938ee4978 2024-11-21T01:46:02,840 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/82323a83e51f44afa07de57938ee4978, entries=150, sequenceid=335, filesize=12.0 K 2024-11-21T01:46:02,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/7380c3c45e6c46fda87272ca0bb6c7a3 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/7380c3c45e6c46fda87272ca0bb6c7a3 2024-11-21T01:46:02,847 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/7380c3c45e6c46fda87272ca0bb6c7a3, entries=150, sequenceid=335, filesize=12.0 K 2024-11-21T01:46:02,847 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 7fd9dee8699da4e84c478aa5cfe68216 in 878ms, sequenceid=335, compaction requested=false 2024-11-21T01:46:02,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2603): Flush status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:46:02,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:02,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-11-21T01:46:02,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=144 2024-11-21T01:46:02,851 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-21T01:46:02,851 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0330 sec 2024-11-21T01:46:02,853 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees in 1.0380 sec 2024-11-21T01:46:02,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-21T01:46:02,953 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:46:02,954 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:46:02,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees 2024-11-21T01:46:02,956 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:46:02,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-21T01:46:02,957 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:46:02,957 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:46:03,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:46:03,059 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fd9dee8699da4e84c478aa5cfe68216 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T01:46:03,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=A 2024-11-21T01:46:03,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:03,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=B 2024-11-21T01:46:03,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:03,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=C 2024-11-21T01:46:03,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:03,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-21T01:46:03,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/21d9b549d7bb4ba1b77f81f99ddf815a is 50, key is test_row_0/A:col10/1732153561999/Put/seqid=0 2024-11-21T01:46:03,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742359_1535 (size=14741) 2024-11-21T01:46:03,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-21T01:46:03,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:03,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:46:03,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:03,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:03,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:03,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:03,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:03,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153623120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:03,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:03,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:03,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:03,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:03,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153623121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:03,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:03,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:03,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:03,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153623193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:03,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:03,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:03,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:03,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:03,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153623195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:03,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:03,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:03,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:03,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:03,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153623232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:03,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:03,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:03,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:03,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:03,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153623242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:03,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:03,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:03,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:03,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-21T01:46:03,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:03,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:46:03,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:03,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:03,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:03,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:03,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-21T01:46:03,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-21T01:46:03,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:03,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:46:03,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:03,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:03,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:03,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:03,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:03,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153623453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:03,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:03,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:03,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:03,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:03,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153623463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:03,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:03,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:03,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:03,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/21d9b549d7bb4ba1b77f81f99ddf815a 2024-11-21T01:46:03,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/8918fef0d4824cc1b3060fdb45aae6a8 is 50, key is test_row_0/B:col10/1732153561999/Put/seqid=0 2024-11-21T01:46:03,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742360_1536 (size=12301) 2024-11-21T01:46:03,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-21T01:46:03,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:03,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:46:03,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:03,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:03,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:03,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:03,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:03,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153623563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:03,578 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b93ada835ef8,41129,1732153434359 for 'test_row_2' in 7fd9dee8699da4e84c478aa5cfe68216 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T01:46:03,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:03,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:03,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:03,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-21T01:46:03,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-21T01:46:03,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:03,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:46:03,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:03,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:03,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:03,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:03,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153623775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:03,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:03,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:03,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:03,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:03,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153623783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:03,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:03,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:03,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:03,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-21T01:46:03,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:03,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:46:03,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:03,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:03,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:03,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:03,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/8918fef0d4824cc1b3060fdb45aae6a8 2024-11-21T01:46:03,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/c2afb07ed5994072b810d627b59d6ed4 is 50, key is test_row_0/C:col10/1732153561999/Put/seqid=0 2024-11-21T01:46:03,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742361_1537 (size=12301) 2024-11-21T01:46:04,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-21T01:46:04,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:04,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:46:04,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:04,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:04,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:04,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:04,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-21T01:46:04,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-21T01:46:04,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:04,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:46:04,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:04,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:04,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:04,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:04,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:04,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153624212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:04,221 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to b93ada835ef8,41129,1732153434359 for 'test_row_0' in 7fd9dee8699da4e84c478aa5cfe68216 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T01:46:04,221 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:04,221 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:04,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:04,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:04,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153624223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:04,233 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b93ada835ef8,41129,1732153434359 for 'test_row_1' in 7fd9dee8699da4e84c478aa5cfe68216 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T01:46:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:04,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:04,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153624292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:04,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:04,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:04,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:04,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:04,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153624303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:04,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:04,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:04,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:04,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/c2afb07ed5994072b810d627b59d6ed4 2024-11-21T01:46:04,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/21d9b549d7bb4ba1b77f81f99ddf815a as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/21d9b549d7bb4ba1b77f81f99ddf815a 2024-11-21T01:46:04,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/21d9b549d7bb4ba1b77f81f99ddf815a, entries=200, sequenceid=350, filesize=14.4 K 2024-11-21T01:46:04,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/8918fef0d4824cc1b3060fdb45aae6a8 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/8918fef0d4824cc1b3060fdb45aae6a8 2024-11-21T01:46:04,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/8918fef0d4824cc1b3060fdb45aae6a8, entries=150, sequenceid=350, filesize=12.0 K 2024-11-21T01:46:04,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/c2afb07ed5994072b810d627b59d6ed4 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/c2afb07ed5994072b810d627b59d6ed4 2024-11-21T01:46:04,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-21T01:46:04,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:04,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:46:04,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:04,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:04,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:04,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:04,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/c2afb07ed5994072b810d627b59d6ed4, entries=150, sequenceid=350, filesize=12.0 K 2024-11-21T01:46:04,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 7fd9dee8699da4e84c478aa5cfe68216 in 1286ms, sequenceid=350, compaction requested=true 2024-11-21T01:46:04,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:46:04,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fd9dee8699da4e84c478aa5cfe68216:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:46:04,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:04,346 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:46:04,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fd9dee8699da4e84c478aa5cfe68216:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:46:04,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:04,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fd9dee8699da4e84c478aa5cfe68216:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:46:04,346 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:46:04,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:46:04,347 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:46:04,347 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 7fd9dee8699da4e84c478aa5cfe68216/B is initiating minor compaction (all files) 2024-11-21T01:46:04,347 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fd9dee8699da4e84c478aa5cfe68216/B in TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:04,347 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/5e1a8528b6ea45f78def8a653941ba7c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/82323a83e51f44afa07de57938ee4978, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/8918fef0d4824cc1b3060fdb45aae6a8] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp, totalSize=36.5 K 2024-11-21T01:46:04,347 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39787 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:46:04,347 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e1a8528b6ea45f78def8a653941ba7c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732153560868 2024-11-21T01:46:04,347 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 7fd9dee8699da4e84c478aa5cfe68216/A is initiating minor compaction (all files) 2024-11-21T01:46:04,347 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fd9dee8699da4e84c478aa5cfe68216/A in TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:04,347 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/6ab79374da00475593735c31fe3a7c02, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/f2b7edfcda3b40a6afc2db4222599d47, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/21d9b549d7bb4ba1b77f81f99ddf815a] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp, totalSize=38.9 K 2024-11-21T01:46:04,348 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 82323a83e51f44afa07de57938ee4978, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732153561353 2024-11-21T01:46:04,348 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ab79374da00475593735c31fe3a7c02, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732153560868 2024-11-21T01:46:04,348 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2b7edfcda3b40a6afc2db4222599d47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732153561353 2024-11-21T01:46:04,348 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 8918fef0d4824cc1b3060fdb45aae6a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732153561999 2024-11-21T01:46:04,348 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21d9b549d7bb4ba1b77f81f99ddf815a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732153561999 2024-11-21T01:46:04,355 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fd9dee8699da4e84c478aa5cfe68216#B#compaction#462 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:04,356 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/9ae28b0751a948d9990506467ce2a3c0 is 50, key is test_row_0/B:col10/1732153561999/Put/seqid=0 2024-11-21T01:46:04,358 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fd9dee8699da4e84c478aa5cfe68216#A#compaction#463 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:04,358 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/75c19ecc3a9243ab915f8bc8072f3836 is 50, key is test_row_0/A:col10/1732153561999/Put/seqid=0 2024-11-21T01:46:04,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742362_1538 (size=12847) 2024-11-21T01:46:04,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742363_1539 (size=12847) 2024-11-21T01:46:04,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-21T01:46:04,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:04,494 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2902): Flushing 7fd9dee8699da4e84c478aa5cfe68216 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-21T01:46:04,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=A 2024-11-21T01:46:04,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:04,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=B 2024-11-21T01:46:04,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:04,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=C 2024-11-21T01:46:04,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:04,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/d8c1d4e1bc794817a9de0c33fa91ec80 is 50, key is test_row_0/A:col10/1732153563118/Put/seqid=0 2024-11-21T01:46:04,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742364_1540 (size=12301) 2024-11-21T01:46:04,767 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/9ae28b0751a948d9990506467ce2a3c0 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/9ae28b0751a948d9990506467ce2a3c0 2024-11-21T01:46:04,778 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fd9dee8699da4e84c478aa5cfe68216/B of 7fd9dee8699da4e84c478aa5cfe68216 into 9ae28b0751a948d9990506467ce2a3c0(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:04,778 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:46:04,778 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., storeName=7fd9dee8699da4e84c478aa5cfe68216/B, priority=13, startTime=1732153564346; duration=0sec 2024-11-21T01:46:04,778 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:46:04,778 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fd9dee8699da4e84c478aa5cfe68216:B 2024-11-21T01:46:04,778 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:46:04,780 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:46:04,780 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 7fd9dee8699da4e84c478aa5cfe68216/C is initiating minor compaction (all files) 2024-11-21T01:46:04,780 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fd9dee8699da4e84c478aa5cfe68216/C in TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:04,780 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/3df195f162444d55a318151eee8e93f8, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/7380c3c45e6c46fda87272ca0bb6c7a3, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/c2afb07ed5994072b810d627b59d6ed4] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp, totalSize=36.5 K 2024-11-21T01:46:04,781 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 3df195f162444d55a318151eee8e93f8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732153560868 2024-11-21T01:46:04,781 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/75c19ecc3a9243ab915f8bc8072f3836 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/75c19ecc3a9243ab915f8bc8072f3836 2024-11-21T01:46:04,781 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 7380c3c45e6c46fda87272ca0bb6c7a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732153561353 2024-11-21T01:46:04,782 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting c2afb07ed5994072b810d627b59d6ed4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732153561999 2024-11-21T01:46:04,789 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fd9dee8699da4e84c478aa5cfe68216#C#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:04,789 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fd9dee8699da4e84c478aa5cfe68216/A of 7fd9dee8699da4e84c478aa5cfe68216 into 75c19ecc3a9243ab915f8bc8072f3836(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:04,789 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:46:04,789 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., storeName=7fd9dee8699da4e84c478aa5cfe68216/A, priority=13, startTime=1732153564345; duration=0sec 2024-11-21T01:46:04,790 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:04,790 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fd9dee8699da4e84c478aa5cfe68216:A 2024-11-21T01:46:04,790 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/5420eb40721f4c0e8393356af7816ce8 is 50, key is test_row_0/C:col10/1732153561999/Put/seqid=0 2024-11-21T01:46:04,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742365_1541 (size=12847) 2024-11-21T01:46:04,803 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/5420eb40721f4c0e8393356af7816ce8 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/5420eb40721f4c0e8393356af7816ce8 2024-11-21T01:46:04,807 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fd9dee8699da4e84c478aa5cfe68216/C of 7fd9dee8699da4e84c478aa5cfe68216 into 5420eb40721f4c0e8393356af7816ce8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:04,807 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:46:04,807 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., storeName=7fd9dee8699da4e84c478aa5cfe68216/C, priority=13, startTime=1732153564346; duration=0sec 2024-11-21T01:46:04,807 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:04,807 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fd9dee8699da4e84c478aa5cfe68216:C 2024-11-21T01:46:04,903 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/d8c1d4e1bc794817a9de0c33fa91ec80 2024-11-21T01:46:04,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/927684819ac94046a8b446e2c98bfb00 is 50, key is test_row_0/B:col10/1732153563118/Put/seqid=0 2024-11-21T01:46:04,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742366_1542 (size=12301) 2024-11-21T01:46:04,923 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/927684819ac94046a8b446e2c98bfb00 2024-11-21T01:46:04,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/6946e347befd4a9881979b5341a485d3 is 50, key is test_row_0/C:col10/1732153563118/Put/seqid=0 2024-11-21T01:46:04,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742367_1543 (size=12301) 2024-11-21T01:46:05,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-21T01:46:05,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:46:05,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:46:05,339 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/6946e347befd4a9881979b5341a485d3 2024-11-21T01:46:05,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/d8c1d4e1bc794817a9de0c33fa91ec80 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/d8c1d4e1bc794817a9de0c33fa91ec80 2024-11-21T01:46:05,348 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/d8c1d4e1bc794817a9de0c33fa91ec80, entries=150, sequenceid=368, filesize=12.0 K 2024-11-21T01:46:05,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/927684819ac94046a8b446e2c98bfb00 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/927684819ac94046a8b446e2c98bfb00 2024-11-21T01:46:05,352 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/927684819ac94046a8b446e2c98bfb00, entries=150, sequenceid=368, filesize=12.0 K 2024-11-21T01:46:05,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/6946e347befd4a9881979b5341a485d3 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/6946e347befd4a9881979b5341a485d3 2024-11-21T01:46:05,357 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/6946e347befd4a9881979b5341a485d3, entries=150, sequenceid=368, filesize=12.0 K 2024-11-21T01:46:05,361 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=53.67 KB/54960 for 7fd9dee8699da4e84c478aa5cfe68216 in 867ms, sequenceid=368, compaction requested=false 2024-11-21T01:46:05,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2603): Flush status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:46:05,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:05,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=146 2024-11-21T01:46:05,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:46:05,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fd9dee8699da4e84c478aa5cfe68216 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T01:46:05,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=A 2024-11-21T01:46:05,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:05,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=B 2024-11-21T01:46:05,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:05,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=C 2024-11-21T01:46:05,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:05,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=146 2024-11-21T01:46:05,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/7540977d4b7741ffbb36b2e791fc82fe is 50, key is test_row_0/A:col10/1732153565353/Put/seqid=0 2024-11-21T01:46:05,370 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-21T01:46:05,370 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4080 sec 2024-11-21T01:46:05,373 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees in 2.4170 sec 2024-11-21T01:46:05,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742368_1544 (size=14741) 2024-11-21T01:46:05,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:05,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153625437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:05,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:05,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:05,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153625439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:05,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:05,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:05,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:05,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:05,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153625553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:05,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:05,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:05,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153625553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:05,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:05,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:05,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:05,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49962 deadline: 1732153625593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:05,601 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b93ada835ef8,41129,1732153434359 for 'test_row_2' in 7fd9dee8699da4e84c478aa5cfe68216 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T01:46:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:05,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/7540977d4b7741ffbb36b2e791fc82fe 2024-11-21T01:46:05,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:05,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153625773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:05,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:05,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:05,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:05,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153625773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:05,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:05,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:05,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:05,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:05,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/be3d2115211442338d06a651ec2154ba is 50, key is test_row_0/B:col10/1732153565353/Put/seqid=0 2024-11-21T01:46:05,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742369_1545 (size=12301) 2024-11-21T01:46:05,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/be3d2115211442338d06a651ec2154ba 2024-11-21T01:46:05,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/421275be6c61449d9f9a40ce61ff5145 is 50, key is test_row_0/C:col10/1732153565353/Put/seqid=0 2024-11-21T01:46:05,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742370_1546 (size=12301) 2024-11-21T01:46:06,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153626093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:06,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:06,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:06,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:06,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153626093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:06,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:06,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:06,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:06,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/421275be6c61449d9f9a40ce61ff5145 2024-11-21T01:46:06,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/7540977d4b7741ffbb36b2e791fc82fe as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/7540977d4b7741ffbb36b2e791fc82fe 2024-11-21T01:46:06,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/7540977d4b7741ffbb36b2e791fc82fe, entries=200, sequenceid=383, filesize=14.4 K 2024-11-21T01:46:06,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/be3d2115211442338d06a651ec2154ba as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/be3d2115211442338d06a651ec2154ba 2024-11-21T01:46:06,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/be3d2115211442338d06a651ec2154ba, entries=150, sequenceid=383, filesize=12.0 K 2024-11-21T01:46:06,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/421275be6c61449d9f9a40ce61ff5145 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/421275be6c61449d9f9a40ce61ff5145 2024-11-21T01:46:06,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/421275be6c61449d9f9a40ce61ff5145, entries=150, sequenceid=383, filesize=12.0 K 2024-11-21T01:46:06,222 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 7fd9dee8699da4e84c478aa5cfe68216 in 860ms, sequenceid=383, compaction requested=true 2024-11-21T01:46:06,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:46:06,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fd9dee8699da4e84c478aa5cfe68216:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:46:06,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:06,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fd9dee8699da4e84c478aa5cfe68216:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:46:06,222 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:46:06,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:06,222 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:46:06,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fd9dee8699da4e84c478aa5cfe68216:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:46:06,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:46:06,223 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:46:06,223 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:46:06,223 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 7fd9dee8699da4e84c478aa5cfe68216/B is initiating minor compaction (all files) 2024-11-21T01:46:06,223 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 7fd9dee8699da4e84c478aa5cfe68216/A is initiating minor compaction (all files) 2024-11-21T01:46:06,223 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fd9dee8699da4e84c478aa5cfe68216/B in TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:06,223 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fd9dee8699da4e84c478aa5cfe68216/A in TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:06,223 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/9ae28b0751a948d9990506467ce2a3c0, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/927684819ac94046a8b446e2c98bfb00, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/be3d2115211442338d06a651ec2154ba] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp, totalSize=36.6 K 2024-11-21T01:46:06,223 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/75c19ecc3a9243ab915f8bc8072f3836, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/d8c1d4e1bc794817a9de0c33fa91ec80, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/7540977d4b7741ffbb36b2e791fc82fe] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp, totalSize=39.0 K 2024-11-21T01:46:06,223 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ae28b0751a948d9990506467ce2a3c0, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732153561999 2024-11-21T01:46:06,223 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75c19ecc3a9243ab915f8bc8072f3836, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732153561999 2024-11-21T01:46:06,224 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8c1d4e1bc794817a9de0c33fa91ec80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732153563115 2024-11-21T01:46:06,224 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 927684819ac94046a8b446e2c98bfb00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732153563115 2024-11-21T01:46:06,225 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting be3d2115211442338d06a651ec2154ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732153565333 2024-11-21T01:46:06,225 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7540977d4b7741ffbb36b2e791fc82fe, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732153565333 2024-11-21T01:46:06,232 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fd9dee8699da4e84c478aa5cfe68216#A#compaction#471 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:06,232 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/7500e72b96ab46f79aac02d7d432a5f5 is 50, key is test_row_0/A:col10/1732153565353/Put/seqid=0 2024-11-21T01:46:06,234 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fd9dee8699da4e84c478aa5cfe68216#B#compaction#472 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:06,234 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/663fa16966514797a1ba40251eca4ebe is 50, key is test_row_0/B:col10/1732153565353/Put/seqid=0 2024-11-21T01:46:06,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742371_1547 (size=12949) 2024-11-21T01:46:06,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742372_1548 (size=12949) 2024-11-21T01:46:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:46:06,252 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fd9dee8699da4e84c478aa5cfe68216 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-21T01:46:06,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=A 2024-11-21T01:46:06,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:46:06,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=B 2024-11-21T01:46:06,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:46:06,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=C 2024-11-21T01:46:06,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:46:06,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/0d06210759df44bfafbb31c4f07a470e is 50, key is test_row_0/A:col10/1732153566250/Put/seqid=0 2024-11-21T01:46:06,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742373_1549 (size=14741) 2024-11-21T01:46:06,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/0d06210759df44bfafbb31c4f07a470e 2024-11-21T01:46:06,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/b31a040e39124fbd83bbb3f7e6a1feab is 50, key is test_row_0/B:col10/1732153566250/Put/seqid=0 2024-11-21T01:46:06,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153626288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:06,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742374_1550 (size=12301) 2024-11-21T01:46:06,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/b31a040e39124fbd83bbb3f7e6a1feab 2024-11-21T01:46:06,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153626290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:06,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:06,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:06,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:06,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/435f700a355f45d096ecbd86cdcda369 is 50, key is test_row_0/C:col10/1732153566250/Put/seqid=0 2024-11-21T01:46:06,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742375_1551 (size=12301) 2024-11-21T01:46:06,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153626404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:06,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:06,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:06,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:06,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153626412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:06,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:06,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:06,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:06,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:06,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153626613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:06,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:06,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:06,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:06,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:06,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153626613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:06,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:06,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:06,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:06,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:06,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153626623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:06,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:06,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:06,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:06,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:06,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153626632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:06,637 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:06,637 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:06,637 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:06,644 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/7500e72b96ab46f79aac02d7d432a5f5 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/7500e72b96ab46f79aac02d7d432a5f5 2024-11-21T01:46:06,648 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/663fa16966514797a1ba40251eca4ebe as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/663fa16966514797a1ba40251eca4ebe 2024-11-21T01:46:06,649 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fd9dee8699da4e84c478aa5cfe68216/A of 7fd9dee8699da4e84c478aa5cfe68216 into 7500e72b96ab46f79aac02d7d432a5f5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:06,649 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:46:06,649 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., storeName=7fd9dee8699da4e84c478aa5cfe68216/A, priority=13, startTime=1732153566222; duration=0sec 2024-11-21T01:46:06,649 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:46:06,649 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fd9dee8699da4e84c478aa5cfe68216:A 2024-11-21T01:46:06,650 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:46:06,652 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:46:06,652 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 7fd9dee8699da4e84c478aa5cfe68216/C is initiating minor compaction (all files) 2024-11-21T01:46:06,652 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fd9dee8699da4e84c478aa5cfe68216/C in TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:06,652 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/5420eb40721f4c0e8393356af7816ce8, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/6946e347befd4a9881979b5341a485d3, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/421275be6c61449d9f9a40ce61ff5145] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp, totalSize=36.6 K 2024-11-21T01:46:06,654 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5420eb40721f4c0e8393356af7816ce8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732153561999 2024-11-21T01:46:06,655 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fd9dee8699da4e84c478aa5cfe68216/B of 7fd9dee8699da4e84c478aa5cfe68216 into 663fa16966514797a1ba40251eca4ebe(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:06,655 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:46:06,655 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., storeName=7fd9dee8699da4e84c478aa5cfe68216/B, priority=13, startTime=1732153566222; duration=0sec 2024-11-21T01:46:06,655 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:06,655 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fd9dee8699da4e84c478aa5cfe68216:B 2024-11-21T01:46:06,656 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6946e347befd4a9881979b5341a485d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732153563115 2024-11-21T01:46:06,656 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 421275be6c61449d9f9a40ce61ff5145, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732153565333 2024-11-21T01:46:06,664 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fd9dee8699da4e84c478aa5cfe68216#C#compaction#476 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:06,664 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/1c6be977a34c4cf3b6ae741531feec8e is 50, key is test_row_0/C:col10/1732153565353/Put/seqid=0 2024-11-21T01:46:06,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742376_1552 (size=12949) 2024-11-21T01:46:06,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/435f700a355f45d096ecbd86cdcda369 2024-11-21T01:46:06,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/0d06210759df44bfafbb31c4f07a470e as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/0d06210759df44bfafbb31c4f07a470e 2024-11-21T01:46:06,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/0d06210759df44bfafbb31c4f07a470e, entries=200, sequenceid=404, filesize=14.4 K 2024-11-21T01:46:06,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/b31a040e39124fbd83bbb3f7e6a1feab as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/b31a040e39124fbd83bbb3f7e6a1feab 2024-11-21T01:46:06,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/b31a040e39124fbd83bbb3f7e6a1feab, entries=150, sequenceid=404, filesize=12.0 K 2024-11-21T01:46:06,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/435f700a355f45d096ecbd86cdcda369 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/435f700a355f45d096ecbd86cdcda369 2024-11-21T01:46:06,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/435f700a355f45d096ecbd86cdcda369, entries=150, sequenceid=404, filesize=12.0 K 2024-11-21T01:46:06,798 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 7fd9dee8699da4e84c478aa5cfe68216 in 546ms, sequenceid=404, compaction requested=false 2024-11-21T01:46:06,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:46:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:46:06,947 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fd9dee8699da4e84c478aa5cfe68216 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T01:46:06,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=A 2024-11-21T01:46:06,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:06,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=B 2024-11-21T01:46:06,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:06,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=C 2024-11-21T01:46:06,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:06,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/4d72fd59ea134010842796a7f0b99efa is 50, key is test_row_0/A:col10/1732153566945/Put/seqid=0 2024-11-21T01:46:06,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742377_1553 (size=14741) 2024-11-21T01:46:06,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/4d72fd59ea134010842796a7f0b99efa 2024-11-21T01:46:06,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/a7264b55739d4525bc0511bde8dc8aba is 50, key is test_row_0/B:col10/1732153566945/Put/seqid=0 2024-11-21T01:46:07,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742378_1554 (size=12301) 2024-11-21T01:46:07,080 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/1c6be977a34c4cf3b6ae741531feec8e as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/1c6be977a34c4cf3b6ae741531feec8e 2024-11-21T01:46:07,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:07,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153627073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:07,085 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:07,085 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:07,085 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:07,087 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fd9dee8699da4e84c478aa5cfe68216/C of 7fd9dee8699da4e84c478aa5cfe68216 into 1c6be977a34c4cf3b6ae741531feec8e(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:07,087 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:46:07,087 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., storeName=7fd9dee8699da4e84c478aa5cfe68216/C, priority=13, startTime=1732153566222; duration=0sec 2024-11-21T01:46:07,088 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:07,088 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fd9dee8699da4e84c478aa5cfe68216:C 2024-11-21T01:46:07,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:07,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153627078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:07,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:07,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:07,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-21T01:46:07,123 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:46:07,124 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:46:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees 2024-11-21T01:46:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-21T01:46:07,126 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=147, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:46:07,126 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=147, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:46:07,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:46:07,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153627193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:07,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153627193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-21T01:46:07,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-21T01:46:07,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:07,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:46:07,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:07,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:07,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:07,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:07,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/a7264b55739d4525bc0511bde8dc8aba 2024-11-21T01:46:07,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/f67213635d0044d2b0800212941490fd is 50, key is test_row_0/C:col10/1732153566945/Put/seqid=0 2024-11-21T01:46:07,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742379_1555 (size=12301) 2024-11-21T01:46:07,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:07,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153627413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:07,421 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:07,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:07,421 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:07,421 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:07,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153627413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:07,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:07,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:07,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:07,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-21T01:46:07,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:07,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:46:07,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:07,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:07,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:07,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-21T01:46:07,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-21T01:46:07,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:07,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:46:07,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:07,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:07,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:07,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:07,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:07,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49946 deadline: 1732153627633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:07,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:07,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50004 deadline: 1732153627633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:07,646 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b93ada835ef8,41129,1732153434359 for 'test_row_0' in 7fd9dee8699da4e84c478aa5cfe68216 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T01:46:07,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:07,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:07,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:07,646 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to b93ada835ef8,41129,1732153434359 for 'test_row_1' in 7fd9dee8699da4e84c478aa5cfe68216 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T01:46:07,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:07,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:07,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:07,731 INFO [Thread-2071 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2071. 2024-11-21T01:46:07,731 DEBUG [Thread-2071 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:46:07,731 DEBUG [Thread-2071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:07,731 DEBUG [Thread-2071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:07,731 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:46:07,733 INFO [Thread-2079 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2079. 2024-11-21T01:46:07,733 DEBUG [Thread-2079 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:46:07,733 DEBUG [Thread-2079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:07,733 DEBUG [Thread-2079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:07,733 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:46:07,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:07,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49988 deadline: 1732153627733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:07,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:07,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:07,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:07,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:07,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49908 deadline: 1732153627733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:07,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 , the old value is region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:07,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:07,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., hostname=b93ada835ef8,41129,1732153434359, seqNum=2 because the exception is null or not the one we care about 2024-11-21T01:46:07,738 INFO [Thread-2073 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2073. 2024-11-21T01:46:07,738 DEBUG [Thread-2073 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:46:07,738 DEBUG [Thread-2073 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:07,738 DEBUG [Thread-2073 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:07,738 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:46:07,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-21T01:46:07,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:07,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:46:07,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:07,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:07,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:07,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:07,749 INFO [Thread-2077 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2077. 2024-11-21T01:46:07,749 DEBUG [Thread-2077 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:46:07,749 INFO [Thread-2075 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2075. 2024-11-21T01:46:07,749 DEBUG [Thread-2077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:07,749 DEBUG [Thread-2075 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:46:07,749 DEBUG [Thread-2077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:07,749 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:46:07,749 DEBUG [Thread-2075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:07,749 DEBUG [Thread-2075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:07,749 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:46:07,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-21T01:46:07,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/f67213635d0044d2b0800212941490fd 2024-11-21T01:46:07,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/4d72fd59ea134010842796a7f0b99efa as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/4d72fd59ea134010842796a7f0b99efa 2024-11-21T01:46:07,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/4d72fd59ea134010842796a7f0b99efa, entries=200, sequenceid=418, filesize=14.4 K 2024-11-21T01:46:07,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/a7264b55739d4525bc0511bde8dc8aba as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/a7264b55739d4525bc0511bde8dc8aba 2024-11-21T01:46:07,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/a7264b55739d4525bc0511bde8dc8aba, entries=150, sequenceid=418, filesize=12.0 K 2024-11-21T01:46:07,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/f67213635d0044d2b0800212941490fd as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/f67213635d0044d2b0800212941490fd 2024-11-21T01:46:07,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/f67213635d0044d2b0800212941490fd, entries=150, sequenceid=418, filesize=12.0 K 2024-11-21T01:46:07,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 7fd9dee8699da4e84c478aa5cfe68216 in 885ms, sequenceid=418, compaction requested=true 2024-11-21T01:46:07,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:46:07,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fd9dee8699da4e84c478aa5cfe68216:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:46:07,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:07,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fd9dee8699da4e84c478aa5cfe68216:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:46:07,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:07,832 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:46:07,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fd9dee8699da4e84c478aa5cfe68216:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:46:07,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:46:07,832 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:46:07,832 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42431 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:46:07,833 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 7fd9dee8699da4e84c478aa5cfe68216/A is initiating minor compaction (all files) 2024-11-21T01:46:07,833 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fd9dee8699da4e84c478aa5cfe68216/A in TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:07,833 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:46:07,833 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/7500e72b96ab46f79aac02d7d432a5f5, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/0d06210759df44bfafbb31c4f07a470e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/4d72fd59ea134010842796a7f0b99efa] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp, totalSize=41.4 K 2024-11-21T01:46:07,833 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): 7fd9dee8699da4e84c478aa5cfe68216/B is initiating minor compaction (all files) 2024-11-21T01:46:07,833 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fd9dee8699da4e84c478aa5cfe68216/B in TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:07,833 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/663fa16966514797a1ba40251eca4ebe, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/b31a040e39124fbd83bbb3f7e6a1feab, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/a7264b55739d4525bc0511bde8dc8aba] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp, totalSize=36.7 K 2024-11-21T01:46:07,833 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7500e72b96ab46f79aac02d7d432a5f5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732153565333 2024-11-21T01:46:07,833 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 663fa16966514797a1ba40251eca4ebe, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732153565333 2024-11-21T01:46:07,833 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d06210759df44bfafbb31c4f07a470e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732153565384 2024-11-21T01:46:07,833 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting b31a040e39124fbd83bbb3f7e6a1feab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732153565384 2024-11-21T01:46:07,834 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d72fd59ea134010842796a7f0b99efa, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1732153566276 2024-11-21T01:46:07,834 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting a7264b55739d4525bc0511bde8dc8aba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1732153566276 2024-11-21T01:46:07,839 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fd9dee8699da4e84c478aa5cfe68216#B#compaction#480 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:07,840 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/7212199ae755432ebc4e0e40ef6bcfd9 is 50, key is test_row_0/B:col10/1732153566945/Put/seqid=0 2024-11-21T01:46:07,841 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fd9dee8699da4e84c478aa5cfe68216#A#compaction#481 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:07,841 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/3a49a9c06e144d2fa846010240b6de87 is 50, key is test_row_0/A:col10/1732153566945/Put/seqid=0 2024-11-21T01:46:07,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742380_1556 (size=13051) 2024-11-21T01:46:07,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742381_1557 (size=13051) 2024-11-21T01:46:07,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-21T01:46:07,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:07,892 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2902): Flushing 7fd9dee8699da4e84c478aa5cfe68216 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T01:46:07,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=A 2024-11-21T01:46:07,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:07,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=B 2024-11-21T01:46:07,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:07,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=C 2024-11-21T01:46:07,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:07,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/f4c061565a444ac99f56452272f99064 is 50, key is test_row_0/A:col10/1732153567075/Put/seqid=0 2024-11-21T01:46:07,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742382_1558 (size=12301) 2024-11-21T01:46:08,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:46:08,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. as already flushing 2024-11-21T01:46:08,244 INFO [Thread-2060 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2060. 2024-11-21T01:46:08,244 DEBUG [Thread-2060 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:46:08,244 INFO [Thread-2062 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2062. 2024-11-21T01:46:08,244 DEBUG [Thread-2060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:08,244 DEBUG [Thread-2062 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:46:08,244 DEBUG [Thread-2062 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:08,244 DEBUG [Thread-2060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:08,245 DEBUG [Thread-2062 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:08,245 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:46:08,245 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:46:08,251 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/7212199ae755432ebc4e0e40ef6bcfd9 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/7212199ae755432ebc4e0e40ef6bcfd9 2024-11-21T01:46:08,251 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/3a49a9c06e144d2fa846010240b6de87 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/3a49a9c06e144d2fa846010240b6de87 2024-11-21T01:46:08,255 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fd9dee8699da4e84c478aa5cfe68216/A of 7fd9dee8699da4e84c478aa5cfe68216 into 3a49a9c06e144d2fa846010240b6de87(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:08,255 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fd9dee8699da4e84c478aa5cfe68216/B of 7fd9dee8699da4e84c478aa5cfe68216 into 7212199ae755432ebc4e0e40ef6bcfd9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:08,255 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:46:08,255 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:46:08,255 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., storeName=7fd9dee8699da4e84c478aa5cfe68216/A, priority=13, startTime=1732153567832; duration=0sec 2024-11-21T01:46:08,255 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., storeName=7fd9dee8699da4e84c478aa5cfe68216/B, priority=13, startTime=1732153567832; duration=0sec 2024-11-21T01:46:08,255 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:46:08,255 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fd9dee8699da4e84c478aa5cfe68216:A 2024-11-21T01:46:08,255 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:08,255 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fd9dee8699da4e84c478aa5cfe68216:B 2024-11-21T01:46:08,255 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:46:08,256 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:46:08,256 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): 7fd9dee8699da4e84c478aa5cfe68216/C is initiating minor compaction (all files) 2024-11-21T01:46:08,256 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fd9dee8699da4e84c478aa5cfe68216/C in TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:08,256 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/1c6be977a34c4cf3b6ae741531feec8e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/435f700a355f45d096ecbd86cdcda369, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/f67213635d0044d2b0800212941490fd] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp, totalSize=36.7 K 2024-11-21T01:46:08,257 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c6be977a34c4cf3b6ae741531feec8e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732153565333 2024-11-21T01:46:08,257 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 435f700a355f45d096ecbd86cdcda369, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732153565384 2024-11-21T01:46:08,257 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting f67213635d0044d2b0800212941490fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1732153566276 2024-11-21T01:46:08,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-21T01:46:08,265 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fd9dee8699da4e84c478aa5cfe68216#C#compaction#483 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:08,265 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/78dbd59be6554d27a218bbf40b48ff33 is 50, key is test_row_0/C:col10/1732153566945/Put/seqid=0 2024-11-21T01:46:08,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742383_1559 (size=13051) 2024-11-21T01:46:08,302 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/f4c061565a444ac99f56452272f99064 2024-11-21T01:46:08,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/76ee82ab21b94b379e752eec2518c7c8 is 50, key is test_row_0/B:col10/1732153567075/Put/seqid=0 2024-11-21T01:46:08,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742384_1560 (size=12301) 2024-11-21T01:46:08,672 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/78dbd59be6554d27a218bbf40b48ff33 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/78dbd59be6554d27a218bbf40b48ff33 2024-11-21T01:46:08,676 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fd9dee8699da4e84c478aa5cfe68216/C of 7fd9dee8699da4e84c478aa5cfe68216 into 78dbd59be6554d27a218bbf40b48ff33(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:08,677 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:46:08,677 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., storeName=7fd9dee8699da4e84c478aa5cfe68216/C, priority=13, startTime=1732153567832; duration=0sec 2024-11-21T01:46:08,677 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:08,677 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fd9dee8699da4e84c478aa5cfe68216:C 2024-11-21T01:46:08,711 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/76ee82ab21b94b379e752eec2518c7c8 2024-11-21T01:46:08,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/92db099629dc484195c12899061f07ba is 50, key is test_row_0/C:col10/1732153567075/Put/seqid=0 2024-11-21T01:46:08,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742385_1561 (size=12301) 2024-11-21T01:46:09,121 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/92db099629dc484195c12899061f07ba 2024-11-21T01:46:09,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/f4c061565a444ac99f56452272f99064 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/f4c061565a444ac99f56452272f99064 2024-11-21T01:46:09,129 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/f4c061565a444ac99f56452272f99064, entries=150, sequenceid=439, filesize=12.0 K 2024-11-21T01:46:09,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/76ee82ab21b94b379e752eec2518c7c8 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/76ee82ab21b94b379e752eec2518c7c8 2024-11-21T01:46:09,133 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/76ee82ab21b94b379e752eec2518c7c8, entries=150, sequenceid=439, filesize=12.0 K 2024-11-21T01:46:09,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/92db099629dc484195c12899061f07ba as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/92db099629dc484195c12899061f07ba 2024-11-21T01:46:09,138 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/92db099629dc484195c12899061f07ba, entries=150, sequenceid=439, filesize=12.0 K 2024-11-21T01:46:09,139 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=13.42 KB/13740 for 7fd9dee8699da4e84c478aa5cfe68216 in 1247ms, sequenceid=439, compaction requested=false 2024-11-21T01:46:09,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2603): Flush status journal for 7fd9dee8699da4e84c478aa5cfe68216: 2024-11-21T01:46:09,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:09,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=148 2024-11-21T01:46:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=148 2024-11-21T01:46:09,142 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-21T01:46:09,143 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0140 sec 2024-11-21T01:46:09,144 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees in 2.0190 sec 2024-11-21T01:46:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-21T01:46:09,284 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:46:09,635 INFO [Thread-2066 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2066. 2024-11-21T01:46:09,635 DEBUG [Thread-2066 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:46:09,635 DEBUG [Thread-2066 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:09,635 DEBUG [Thread-2066 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:09,635 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:46:09,654 INFO [Thread-2064 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2064. 2024-11-21T01:46:09,655 DEBUG [Thread-2064 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:46:09,655 DEBUG [Thread-2064 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:09,655 DEBUG [Thread-2064 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:09,655 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:46:09,665 INFO [Thread-2068 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2068. 2024-11-21T01:46:09,665 DEBUG [Thread-2068 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:46:09,665 DEBUG [Thread-2068 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:09,665 DEBUG [Thread-2068 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:09,665 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:46:09,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T01:46:09,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-21T01:46:09,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-21T01:46:09,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-21T01:46:09,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-21T01:46:09,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-21T01:46:09,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T01:46:09,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T01:46:09,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 936 2024-11-21T01:46:09,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2802 rows 2024-11-21T01:46:09,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 926 2024-11-21T01:46:09,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2778 rows 2024-11-21T01:46:09,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 942 2024-11-21T01:46:09,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2817 rows 2024-11-21T01:46:09,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 931 2024-11-21T01:46:09,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2789 rows 2024-11-21T01:46:09,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 933 2024-11-21T01:46:09,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2798 rows 2024-11-21T01:46:09,666 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T01:46:09,666 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T01:46:09,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:09,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:09,666 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:46:09,667 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-21T01:46:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T01:46:09,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-21T01:46:09,670 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732153569670"}]},"ts":"1732153569670"} 2024-11-21T01:46:09,671 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T01:46:09,671 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T01:46:09,672 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T01:46:09,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fd9dee8699da4e84c478aa5cfe68216, UNASSIGN}] 2024-11-21T01:46:09,674 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fd9dee8699da4e84c478aa5cfe68216, UNASSIGN 2024-11-21T01:46:09,675 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=7fd9dee8699da4e84c478aa5cfe68216, regionState=CLOSING, regionLocation=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:09,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fd9dee8699da4e84c478aa5cfe68216, UNASSIGN because future has completed 2024-11-21T01:46:09,676 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T01:46:09,677 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359}] 2024-11-21T01:46:09,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-21T01:46:09,829 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(122): Close 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:46:09,829 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T01:46:09,829 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1722): Closing 7fd9dee8699da4e84c478aa5cfe68216, disabling compactions & flushes 2024-11-21T01:46:09,829 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:09,829 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:09,829 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. after waiting 0 ms 2024-11-21T01:46:09,829 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:09,829 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(2902): Flushing 7fd9dee8699da4e84c478aa5cfe68216 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-21T01:46:09,829 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=A 2024-11-21T01:46:09,829 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:09,829 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=B 2024-11-21T01:46:09,829 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:09,829 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7fd9dee8699da4e84c478aa5cfe68216, store=C 2024-11-21T01:46:09,829 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:09,834 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/1b60d1bf0a1f4ceabd3b21d1755dde7c is 50, key is test_row_0/A:col10/1732153569663/Put/seqid=0 2024-11-21T01:46:09,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742386_1562 (size=12301) 2024-11-21T01:46:09,841 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/1b60d1bf0a1f4ceabd3b21d1755dde7c 2024-11-21T01:46:09,847 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/99eb6674893546e09d785e2605341548 is 50, key is test_row_0/B:col10/1732153569663/Put/seqid=0 2024-11-21T01:46:09,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742387_1563 (size=12301) 2024-11-21T01:46:09,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-21T01:46:10,251 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/99eb6674893546e09d785e2605341548 2024-11-21T01:46:10,258 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/8cdf30f96d1f42778c306a56a759a2eb is 50, key is test_row_0/C:col10/1732153569663/Put/seqid=0 2024-11-21T01:46:10,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742388_1564 (size=12301) 2024-11-21T01:46:10,264 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/8cdf30f96d1f42778c306a56a759a2eb 2024-11-21T01:46:10,267 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/A/1b60d1bf0a1f4ceabd3b21d1755dde7c as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/1b60d1bf0a1f4ceabd3b21d1755dde7c 2024-11-21T01:46:10,270 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/1b60d1bf0a1f4ceabd3b21d1755dde7c, entries=150, sequenceid=450, filesize=12.0 K 2024-11-21T01:46:10,271 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/B/99eb6674893546e09d785e2605341548 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/99eb6674893546e09d785e2605341548 2024-11-21T01:46:10,274 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/99eb6674893546e09d785e2605341548, entries=150, sequenceid=450, filesize=12.0 K 2024-11-21T01:46:10,274 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/.tmp/C/8cdf30f96d1f42778c306a56a759a2eb as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/8cdf30f96d1f42778c306a56a759a2eb 2024-11-21T01:46:10,277 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/8cdf30f96d1f42778c306a56a759a2eb, entries=150, sequenceid=450, filesize=12.0 K 2024-11-21T01:46:10,278 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 7fd9dee8699da4e84c478aa5cfe68216 in 449ms, sequenceid=450, compaction requested=true 2024-11-21T01:46:10,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/180751cadac140faaa1d2add6a1e33e5, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/2abb11d59d9f4ddbb70eaaaa21a28db2, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/a816e8ed79b94bb49234d03531c17d20, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/f94ae44c6beb458aa9680556755937ba, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/c418c12c935a4c7483f3e5adf1882e2c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/2d10e65277794c00bc23d5790b5982d4, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/d91b3177e3de49a691e3aee194c6a0f5, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/15c7c787f06c48f4883e190f459ddf65, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/2714c72c47774ae2a59e5aa6a4e12849, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/2f35986e024a475b824378fd77858c90, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/547baa89ea4c43919e218ee449a892dc, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/c4a5dce64bd14fa095b07e86451a2787, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/9fa45c2d245e4443a49de321358f0fa8, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/6ab79374da00475593735c31fe3a7c02, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/f2b7edfcda3b40a6afc2db4222599d47, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/21d9b549d7bb4ba1b77f81f99ddf815a, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/75c19ecc3a9243ab915f8bc8072f3836, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/d8c1d4e1bc794817a9de0c33fa91ec80, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/7540977d4b7741ffbb36b2e791fc82fe, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/7500e72b96ab46f79aac02d7d432a5f5, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/0d06210759df44bfafbb31c4f07a470e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/4d72fd59ea134010842796a7f0b99efa] to archive 2024-11-21T01:46:10,280 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T01:46:10,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/180751cadac140faaa1d2add6a1e33e5 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/180751cadac140faaa1d2add6a1e33e5 2024-11-21T01:46:10,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/2abb11d59d9f4ddbb70eaaaa21a28db2 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/2abb11d59d9f4ddbb70eaaaa21a28db2 2024-11-21T01:46:10,283 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/a816e8ed79b94bb49234d03531c17d20 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/a816e8ed79b94bb49234d03531c17d20 2024-11-21T01:46:10,285 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/f94ae44c6beb458aa9680556755937ba to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/f94ae44c6beb458aa9680556755937ba 2024-11-21T01:46:10,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/c418c12c935a4c7483f3e5adf1882e2c to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/c418c12c935a4c7483f3e5adf1882e2c 2024-11-21T01:46:10,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/2d10e65277794c00bc23d5790b5982d4 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/2d10e65277794c00bc23d5790b5982d4 2024-11-21T01:46:10,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/d91b3177e3de49a691e3aee194c6a0f5 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/d91b3177e3de49a691e3aee194c6a0f5 2024-11-21T01:46:10,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-21T01:46:10,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/15c7c787f06c48f4883e190f459ddf65 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/15c7c787f06c48f4883e190f459ddf65 2024-11-21T01:46:10,295 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/2714c72c47774ae2a59e5aa6a4e12849 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/2714c72c47774ae2a59e5aa6a4e12849 2024-11-21T01:46:10,296 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/2f35986e024a475b824378fd77858c90 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/2f35986e024a475b824378fd77858c90 2024-11-21T01:46:10,297 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/547baa89ea4c43919e218ee449a892dc to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/547baa89ea4c43919e218ee449a892dc 2024-11-21T01:46:10,298 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/c4a5dce64bd14fa095b07e86451a2787 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/c4a5dce64bd14fa095b07e86451a2787 2024-11-21T01:46:10,300 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/9fa45c2d245e4443a49de321358f0fa8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/9fa45c2d245e4443a49de321358f0fa8 2024-11-21T01:46:10,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/6ab79374da00475593735c31fe3a7c02 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/6ab79374da00475593735c31fe3a7c02 2024-11-21T01:46:10,302 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/f2b7edfcda3b40a6afc2db4222599d47 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/f2b7edfcda3b40a6afc2db4222599d47 2024-11-21T01:46:10,303 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/21d9b549d7bb4ba1b77f81f99ddf815a to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/21d9b549d7bb4ba1b77f81f99ddf815a 2024-11-21T01:46:10,304 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/75c19ecc3a9243ab915f8bc8072f3836 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/75c19ecc3a9243ab915f8bc8072f3836 2024-11-21T01:46:10,306 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/d8c1d4e1bc794817a9de0c33fa91ec80 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/d8c1d4e1bc794817a9de0c33fa91ec80 2024-11-21T01:46:10,307 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/7540977d4b7741ffbb36b2e791fc82fe to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/7540977d4b7741ffbb36b2e791fc82fe 2024-11-21T01:46:10,309 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/7500e72b96ab46f79aac02d7d432a5f5 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/7500e72b96ab46f79aac02d7d432a5f5 2024-11-21T01:46:10,310 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/0d06210759df44bfafbb31c4f07a470e to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/0d06210759df44bfafbb31c4f07a470e 2024-11-21T01:46:10,311 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/4d72fd59ea134010842796a7f0b99efa to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/4d72fd59ea134010842796a7f0b99efa 2024-11-21T01:46:10,325 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/9e004e4e620a43fba94885919e1ae245, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/5a01435af0e142b4b732b5e3183fb362, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/bf8ef3853a784e93847acd58c2929a28, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/77edf1cf02d2465d972befd3db56fb7f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/055487325a534ee5b14a317235591439, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/32700fbc27fd4289bbeaa35168e427e8, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/3d90bf854509439db82b1d8c678880ab, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/7b5a988fc6f24c148d99fe63cdcfc3cf, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/6f3feb3c8bea4daa94a90bc5b4217a0f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/f809396577464406a5c1ad30a9a43dad, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/d0352424bf2d440cb0790284159d2715, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/8fd61161156143d5b0de329a6ca6d99c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/5e1a8528b6ea45f78def8a653941ba7c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/3b7d81d77e9a42ea822c59633227f28a, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/82323a83e51f44afa07de57938ee4978, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/9ae28b0751a948d9990506467ce2a3c0, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/8918fef0d4824cc1b3060fdb45aae6a8, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/927684819ac94046a8b446e2c98bfb00, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/663fa16966514797a1ba40251eca4ebe, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/be3d2115211442338d06a651ec2154ba, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/b31a040e39124fbd83bbb3f7e6a1feab, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/a7264b55739d4525bc0511bde8dc8aba] to archive 2024-11-21T01:46:10,327 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T01:46:10,354 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/9e004e4e620a43fba94885919e1ae245 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/9e004e4e620a43fba94885919e1ae245 2024-11-21T01:46:10,358 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/5a01435af0e142b4b732b5e3183fb362 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/5a01435af0e142b4b732b5e3183fb362 2024-11-21T01:46:10,370 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/bf8ef3853a784e93847acd58c2929a28 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/bf8ef3853a784e93847acd58c2929a28 2024-11-21T01:46:10,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/77edf1cf02d2465d972befd3db56fb7f to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/77edf1cf02d2465d972befd3db56fb7f 2024-11-21T01:46:10,379 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/055487325a534ee5b14a317235591439 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/055487325a534ee5b14a317235591439 2024-11-21T01:46:10,380 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/32700fbc27fd4289bbeaa35168e427e8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/32700fbc27fd4289bbeaa35168e427e8 2024-11-21T01:46:10,384 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/3d90bf854509439db82b1d8c678880ab to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/3d90bf854509439db82b1d8c678880ab 2024-11-21T01:46:10,385 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/7b5a988fc6f24c148d99fe63cdcfc3cf to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/7b5a988fc6f24c148d99fe63cdcfc3cf 2024-11-21T01:46:10,386 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/6f3feb3c8bea4daa94a90bc5b4217a0f to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/6f3feb3c8bea4daa94a90bc5b4217a0f 2024-11-21T01:46:10,387 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/f809396577464406a5c1ad30a9a43dad to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/f809396577464406a5c1ad30a9a43dad 2024-11-21T01:46:10,388 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/d0352424bf2d440cb0790284159d2715 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/d0352424bf2d440cb0790284159d2715 2024-11-21T01:46:10,389 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/8fd61161156143d5b0de329a6ca6d99c to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/8fd61161156143d5b0de329a6ca6d99c 2024-11-21T01:46:10,390 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/5e1a8528b6ea45f78def8a653941ba7c to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/5e1a8528b6ea45f78def8a653941ba7c 2024-11-21T01:46:10,391 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/3b7d81d77e9a42ea822c59633227f28a to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/3b7d81d77e9a42ea822c59633227f28a 2024-11-21T01:46:10,392 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/82323a83e51f44afa07de57938ee4978 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/82323a83e51f44afa07de57938ee4978 2024-11-21T01:46:10,393 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/9ae28b0751a948d9990506467ce2a3c0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/9ae28b0751a948d9990506467ce2a3c0 2024-11-21T01:46:10,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/8918fef0d4824cc1b3060fdb45aae6a8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/8918fef0d4824cc1b3060fdb45aae6a8 2024-11-21T01:46:10,396 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/927684819ac94046a8b446e2c98bfb00 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/927684819ac94046a8b446e2c98bfb00 2024-11-21T01:46:10,397 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/663fa16966514797a1ba40251eca4ebe to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/663fa16966514797a1ba40251eca4ebe 2024-11-21T01:46:10,398 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/be3d2115211442338d06a651ec2154ba to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/be3d2115211442338d06a651ec2154ba 2024-11-21T01:46:10,399 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/b31a040e39124fbd83bbb3f7e6a1feab to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/b31a040e39124fbd83bbb3f7e6a1feab 2024-11-21T01:46:10,400 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/a7264b55739d4525bc0511bde8dc8aba to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/a7264b55739d4525bc0511bde8dc8aba 2024-11-21T01:46:10,401 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/db41156d3e30425284a693e5b1980ce8, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/a185d36597a14e84af246c34467b3851, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/3286fc15af1b48bb93f8039b47f522ae, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/ead5230fb3394dc4bf90ed38080e9a3e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/9d9838e46aff4f5084e869eff16108bb, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/7a8c14b0756149d5825e237e011da242, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/643f7d8858684cf9b299510d2403d34c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/ff286b1a867e436f8cfd927b38a17f89, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/69ac80d25e8c49d78a44fc67b6a6cc8f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/7bddd417a45444468f702fb31f34b1fb, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/cbc3fadf8ab142a6bbde875acfe2034d, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/e8cf65e1de9e4035a7cf7a98c3391cc0, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/3df195f162444d55a318151eee8e93f8, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/566c73149bf54f0188ae9efb55bc2174, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/7380c3c45e6c46fda87272ca0bb6c7a3, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/5420eb40721f4c0e8393356af7816ce8, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/c2afb07ed5994072b810d627b59d6ed4, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/6946e347befd4a9881979b5341a485d3, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/1c6be977a34c4cf3b6ae741531feec8e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/421275be6c61449d9f9a40ce61ff5145, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/435f700a355f45d096ecbd86cdcda369, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/f67213635d0044d2b0800212941490fd] to archive 2024-11-21T01:46:10,402 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T01:46:10,404 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/db41156d3e30425284a693e5b1980ce8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/db41156d3e30425284a693e5b1980ce8 2024-11-21T01:46:10,405 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/a185d36597a14e84af246c34467b3851 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/a185d36597a14e84af246c34467b3851 2024-11-21T01:46:10,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/3286fc15af1b48bb93f8039b47f522ae to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/3286fc15af1b48bb93f8039b47f522ae 2024-11-21T01:46:10,407 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/ead5230fb3394dc4bf90ed38080e9a3e to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/ead5230fb3394dc4bf90ed38080e9a3e 2024-11-21T01:46:10,408 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/9d9838e46aff4f5084e869eff16108bb to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/9d9838e46aff4f5084e869eff16108bb 2024-11-21T01:46:10,410 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/7a8c14b0756149d5825e237e011da242 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/7a8c14b0756149d5825e237e011da242 2024-11-21T01:46:10,411 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/643f7d8858684cf9b299510d2403d34c to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/643f7d8858684cf9b299510d2403d34c 2024-11-21T01:46:10,412 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/ff286b1a867e436f8cfd927b38a17f89 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/ff286b1a867e436f8cfd927b38a17f89 2024-11-21T01:46:10,414 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/69ac80d25e8c49d78a44fc67b6a6cc8f to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/69ac80d25e8c49d78a44fc67b6a6cc8f 2024-11-21T01:46:10,415 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/7bddd417a45444468f702fb31f34b1fb to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/7bddd417a45444468f702fb31f34b1fb 2024-11-21T01:46:10,416 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/cbc3fadf8ab142a6bbde875acfe2034d to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/cbc3fadf8ab142a6bbde875acfe2034d 2024-11-21T01:46:10,417 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/e8cf65e1de9e4035a7cf7a98c3391cc0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/e8cf65e1de9e4035a7cf7a98c3391cc0 2024-11-21T01:46:10,418 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/3df195f162444d55a318151eee8e93f8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/3df195f162444d55a318151eee8e93f8 2024-11-21T01:46:10,419 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/566c73149bf54f0188ae9efb55bc2174 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/566c73149bf54f0188ae9efb55bc2174 2024-11-21T01:46:10,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/7380c3c45e6c46fda87272ca0bb6c7a3 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/7380c3c45e6c46fda87272ca0bb6c7a3 2024-11-21T01:46:10,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/5420eb40721f4c0e8393356af7816ce8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/5420eb40721f4c0e8393356af7816ce8 2024-11-21T01:46:10,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/c2afb07ed5994072b810d627b59d6ed4 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/c2afb07ed5994072b810d627b59d6ed4 2024-11-21T01:46:10,423 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/6946e347befd4a9881979b5341a485d3 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/6946e347befd4a9881979b5341a485d3 2024-11-21T01:46:10,424 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/1c6be977a34c4cf3b6ae741531feec8e to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/1c6be977a34c4cf3b6ae741531feec8e 2024-11-21T01:46:10,424 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/421275be6c61449d9f9a40ce61ff5145 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/421275be6c61449d9f9a40ce61ff5145 2024-11-21T01:46:10,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/435f700a355f45d096ecbd86cdcda369 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/435f700a355f45d096ecbd86cdcda369 2024-11-21T01:46:10,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/f67213635d0044d2b0800212941490fd to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/f67213635d0044d2b0800212941490fd 2024-11-21T01:46:10,430 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/recovered.edits/453.seqid, newMaxSeqId=453, maxSeqId=1 2024-11-21T01:46:10,430 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. 2024-11-21T01:46:10,431 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1676): Region close journal for 7fd9dee8699da4e84c478aa5cfe68216: Waiting for close lock at 1732153569829Running coprocessor pre-close hooks at 1732153569829Disabling compacts and flushes for region at 1732153569829Disabling writes for close at 1732153569829Obtaining lock to block concurrent updates at 1732153569829Preparing flush snapshotting stores in 7fd9dee8699da4e84c478aa5cfe68216 at 1732153569829Finished memstore snapshotting TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732153569830 (+1 ms)Flushing stores of TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216. at 1732153569830Flushing 7fd9dee8699da4e84c478aa5cfe68216/A: creating writer at 1732153569830Flushing 7fd9dee8699da4e84c478aa5cfe68216/A: appending metadata at 1732153569833 (+3 ms)Flushing 7fd9dee8699da4e84c478aa5cfe68216/A: closing flushed file at 1732153569833Flushing 7fd9dee8699da4e84c478aa5cfe68216/B: creating writer at 1732153569845 (+12 ms)Flushing 7fd9dee8699da4e84c478aa5cfe68216/B: appending metadata at 1732153569847 (+2 ms)Flushing 7fd9dee8699da4e84c478aa5cfe68216/B: closing flushed file at 1732153569847Flushing 7fd9dee8699da4e84c478aa5cfe68216/C: creating writer at 1732153570255 (+408 ms)Flushing 7fd9dee8699da4e84c478aa5cfe68216/C: appending metadata at 1732153570258 (+3 ms)Flushing 7fd9dee8699da4e84c478aa5cfe68216/C: closing flushed file at 1732153570258Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64b5a5a3: reopening flushed file at 1732153570266 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@428249ca: reopening flushed file at 1732153570270 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4269b787: reopening flushed file at 1732153570274 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 7fd9dee8699da4e84c478aa5cfe68216 in 449ms, sequenceid=450, compaction requested=true at 1732153570278 (+4 ms)Writing region close event to WAL at 1732153570427 (+149 ms)Running coprocessor post-close hooks at 1732153570430 (+3 ms)Closed at 1732153570430 2024-11-21T01:46:10,432 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(157): Closed 7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:46:10,433 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=7fd9dee8699da4e84c478aa5cfe68216, regionState=CLOSED 2024-11-21T01:46:10,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 because future has completed 2024-11-21T01:46:10,436 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-21T01:46:10,436 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; CloseRegionProcedure 7fd9dee8699da4e84c478aa5cfe68216, server=b93ada835ef8,41129,1732153434359 in 757 msec 2024-11-21T01:46:10,437 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-21T01:46:10,437 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fd9dee8699da4e84c478aa5cfe68216, UNASSIGN in 763 msec 2024-11-21T01:46:10,440 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-21T01:46:10,440 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 766 msec 2024-11-21T01:46:10,443 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732153570442"}]},"ts":"1732153570442"} 2024-11-21T01:46:10,444 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T01:46:10,444 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T01:46:10,446 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 778 msec 2024-11-21T01:46:10,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-21T01:46:10,803 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:46:10,804 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-21T01:46:10,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=153, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:46:10,806 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=153, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:46:10,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-11-21T01:46:10,806 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=153, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:46:10,809 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:46:10,811 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A, FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B, FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C, FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/recovered.edits] 2024-11-21T01:46:10,814 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/1b60d1bf0a1f4ceabd3b21d1755dde7c to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/1b60d1bf0a1f4ceabd3b21d1755dde7c 2024-11-21T01:46:10,815 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/3a49a9c06e144d2fa846010240b6de87 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/3a49a9c06e144d2fa846010240b6de87 2024-11-21T01:46:10,816 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/f4c061565a444ac99f56452272f99064 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/A/f4c061565a444ac99f56452272f99064 2024-11-21T01:46:10,817 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/7212199ae755432ebc4e0e40ef6bcfd9 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/7212199ae755432ebc4e0e40ef6bcfd9 2024-11-21T01:46:10,818 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/76ee82ab21b94b379e752eec2518c7c8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/76ee82ab21b94b379e752eec2518c7c8 2024-11-21T01:46:10,820 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/99eb6674893546e09d785e2605341548 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/B/99eb6674893546e09d785e2605341548 2024-11-21T01:46:10,822 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/78dbd59be6554d27a218bbf40b48ff33 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/78dbd59be6554d27a218bbf40b48ff33 2024-11-21T01:46:10,823 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/8cdf30f96d1f42778c306a56a759a2eb to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/8cdf30f96d1f42778c306a56a759a2eb 2024-11-21T01:46:10,824 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/92db099629dc484195c12899061f07ba to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/C/92db099629dc484195c12899061f07ba 2024-11-21T01:46:10,826 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/recovered.edits/453.seqid to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216/recovered.edits/453.seqid 2024-11-21T01:46:10,826 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/7fd9dee8699da4e84c478aa5cfe68216 2024-11-21T01:46:10,826 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T01:46:10,828 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=153, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:46:10,830 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T01:46:10,832 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T01:46:10,833 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=153, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:46:10,833 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T01:46:10,834 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732153570833"}]},"ts":"9223372036854775807"} 2024-11-21T01:46:10,835 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T01:46:10,835 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 7fd9dee8699da4e84c478aa5cfe68216, NAME => 'TestAcidGuarantees,,1732153545522.7fd9dee8699da4e84c478aa5cfe68216.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T01:46:10,836 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T01:46:10,836 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732153570836"}]},"ts":"9223372036854775807"} 2024-11-21T01:46:10,837 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T01:46:10,838 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=153, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:46:10,839 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 34 msec 2024-11-21T01:46:10,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-11-21T01:46:10,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T01:46:10,918 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:46:10,926 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 250), OpenFileDescriptor=449 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=619 (was 577) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3323 (was 2409) - AvailableMemoryMB LEAK? - 2024-11-21T01:46:10,937 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=619, ProcessCount=11, AvailableMemoryMB=3322 2024-11-21T01:46:10,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T01:46:10,938 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T01:46:10,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T01:46:10,940 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=154, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T01:46:10,941 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:10,941 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 154 2024-11-21T01:46:10,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T01:46:10,941 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=154, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T01:46:10,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742389_1565 (size=1061) 2024-11-21T01:46:11,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T01:46:11,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T01:46:11,349 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d 2024-11-21T01:46:11,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742390_1566 (size=53) 2024-11-21T01:46:11,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T01:46:11,769 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:46:11,769 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f1185a7567ce5119c4e675690fbe9d2b, disabling compactions & flushes 2024-11-21T01:46:11,769 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:11,769 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:11,769 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. after waiting 0 ms 2024-11-21T01:46:11,769 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:11,769 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:11,769 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f1185a7567ce5119c4e675690fbe9d2b: Waiting for close lock at 1732153571769Disabling compacts and flushes for region at 1732153571769Disabling writes for close at 1732153571769Writing region close event to WAL at 1732153571769Closed at 1732153571769 2024-11-21T01:46:11,770 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=154, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T01:46:11,771 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732153571770"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732153571770"}]},"ts":"1732153571770"} 2024-11-21T01:46:11,772 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T01:46:11,773 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=154, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T01:46:11,773 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732153571773"}]},"ts":"1732153571773"} 2024-11-21T01:46:11,774 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T01:46:11,775 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f1185a7567ce5119c4e675690fbe9d2b, ASSIGN}] 2024-11-21T01:46:11,776 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f1185a7567ce5119c4e675690fbe9d2b, ASSIGN 2024-11-21T01:46:11,777 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f1185a7567ce5119c4e675690fbe9d2b, ASSIGN; state=OFFLINE, location=b93ada835ef8,41129,1732153434359; forceNewPlan=false, retain=false 2024-11-21T01:46:11,927 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=f1185a7567ce5119c4e675690fbe9d2b, regionState=OPENING, regionLocation=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:11,929 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f1185a7567ce5119c4e675690fbe9d2b, ASSIGN because future has completed 2024-11-21T01:46:11,929 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359}] 2024-11-21T01:46:12,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T01:46:12,084 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:12,084 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(7752): Opening region: {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} 2024-11-21T01:46:12,084 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:12,084 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:46:12,084 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(7794): checking encryption for f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:12,084 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(7797): checking classloading for f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:12,086 INFO [StoreOpener-f1185a7567ce5119c4e675690fbe9d2b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:12,087 INFO [StoreOpener-f1185a7567ce5119c4e675690fbe9d2b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T01:46:12,087 INFO [StoreOpener-f1185a7567ce5119c4e675690fbe9d2b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1185a7567ce5119c4e675690fbe9d2b columnFamilyName A 2024-11-21T01:46:12,087 DEBUG [StoreOpener-f1185a7567ce5119c4e675690fbe9d2b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:12,087 INFO [StoreOpener-f1185a7567ce5119c4e675690fbe9d2b-1 {}] regionserver.HStore(327): Store=f1185a7567ce5119c4e675690fbe9d2b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:46:12,087 INFO [StoreOpener-f1185a7567ce5119c4e675690fbe9d2b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:12,088 INFO [StoreOpener-f1185a7567ce5119c4e675690fbe9d2b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T01:46:12,089 INFO [StoreOpener-f1185a7567ce5119c4e675690fbe9d2b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1185a7567ce5119c4e675690fbe9d2b columnFamilyName B 2024-11-21T01:46:12,089 DEBUG [StoreOpener-f1185a7567ce5119c4e675690fbe9d2b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:12,089 INFO [StoreOpener-f1185a7567ce5119c4e675690fbe9d2b-1 {}] regionserver.HStore(327): Store=f1185a7567ce5119c4e675690fbe9d2b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:46:12,089 INFO [StoreOpener-f1185a7567ce5119c4e675690fbe9d2b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:12,090 INFO [StoreOpener-f1185a7567ce5119c4e675690fbe9d2b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T01:46:12,090 INFO [StoreOpener-f1185a7567ce5119c4e675690fbe9d2b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1185a7567ce5119c4e675690fbe9d2b columnFamilyName C 2024-11-21T01:46:12,090 DEBUG [StoreOpener-f1185a7567ce5119c4e675690fbe9d2b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:12,090 INFO [StoreOpener-f1185a7567ce5119c4e675690fbe9d2b-1 {}] regionserver.HStore(327): Store=f1185a7567ce5119c4e675690fbe9d2b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:46:12,091 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:12,091 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(1038): replaying wal for f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:12,091 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:12,091 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:12,092 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(1048): stopping wal replay for f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:12,092 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(1060): Cleaning up temporary data for f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:12,093 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T01:46:12,094 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(1093): writing seq id for f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:12,096 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T01:46:12,096 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(1114): Opened f1185a7567ce5119c4e675690fbe9d2b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70606957, jitterRate=0.05212564766407013}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T01:46:12,096 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:12,097 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(1006): Region open journal for f1185a7567ce5119c4e675690fbe9d2b: Running coprocessor pre-open hook at 1732153572084Writing region info on filesystem at 1732153572084Initializing all the Stores at 1732153572085 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732153572085Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732153572085Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732153572085Cleaning up temporary data from old regions at 1732153572092 (+7 ms)Running coprocessor post-open hooks at 1732153572096 (+4 ms)Region opened successfully at 1732153572097 (+1 ms) 2024-11-21T01:46:12,098 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., pid=156, masterSystemTime=1732153572081 2024-11-21T01:46:12,099 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:12,099 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:12,100 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=f1185a7567ce5119c4e675690fbe9d2b, regionState=OPEN, openSeqNum=2, regionLocation=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:12,101 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=156, ppid=155, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 because future has completed 2024-11-21T01:46:12,103 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=156, resume processing ppid=155 2024-11-21T01:46:12,104 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, ppid=155, state=SUCCESS, hasLock=false; OpenRegionProcedure f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 in 173 msec 2024-11-21T01:46:12,106 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-21T01:46:12,106 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f1185a7567ce5119c4e675690fbe9d2b, ASSIGN in 329 msec 2024-11-21T01:46:12,106 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=154, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T01:46:12,106 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732153572106"}]},"ts":"1732153572106"} 2024-11-21T01:46:12,108 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T01:46:12,108 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=154, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T01:46:12,110 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1690 sec 2024-11-21T01:46:13,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T01:46:13,094 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:46:13,095 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@355ff781, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,095 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:46:13,095 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:46:13,096 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:46:13,096 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:46:13,096 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:46:13,096 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63388b3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,097 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:46:13,097 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:46:13,097 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:13,098 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34804, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:46:13,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bf36cee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,099 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:46:13,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:46:13,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:46:13,100 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48256, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:46:13,101 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T01:46:13,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is b93ada835ef8,36607,1732153433607 2024-11-21T01:46:13,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@238d542a 2024-11-21T01:46:13,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T01:46:13,103 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34814, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T01:46:13,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T01:46:13,104 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T01:46:13,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=157, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-21T01:46:13,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742391_1567 (size=1097) 2024-11-21T01:46:13,120 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-21T01:46:13,120 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-21T01:46:13,122 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=158, ppid=157, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T01:46:13,124 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f1185a7567ce5119c4e675690fbe9d2b, REOPEN/MOVE}] 2024-11-21T01:46:13,125 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=159, ppid=158, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f1185a7567ce5119c4e675690fbe9d2b, REOPEN/MOVE 2024-11-21T01:46:13,126 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=159 updating hbase:meta row=f1185a7567ce5119c4e675690fbe9d2b, regionState=CLOSING, regionLocation=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:13,128 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=159, ppid=158, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f1185a7567ce5119c4e675690fbe9d2b, REOPEN/MOVE because future has completed 2024-11-21T01:46:13,128 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T01:46:13,128 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE, hasLock=false; CloseRegionProcedure f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359}] 2024-11-21T01:46:13,280 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] handler.UnassignRegionHandler(122): Close f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:13,280 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T01:46:13,280 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] regionserver.HRegion(1722): Closing f1185a7567ce5119c4e675690fbe9d2b, disabling compactions & flushes 2024-11-21T01:46:13,280 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:13,280 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:13,280 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. after waiting 0 ms 2024-11-21T01:46:13,280 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:13,284 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T01:46:13,284 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:13,284 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] regionserver.HRegion(1676): Region close journal for f1185a7567ce5119c4e675690fbe9d2b: Waiting for close lock at 1732153573280Running coprocessor pre-close hooks at 1732153573280Disabling compacts and flushes for region at 1732153573280Disabling writes for close at 1732153573280Writing region close event to WAL at 1732153573281 (+1 ms)Running coprocessor post-close hooks at 1732153573284 (+3 ms)Closed at 1732153573284 2024-11-21T01:46:13,284 WARN [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] regionserver.HRegionServer(3299): Not adding moved region record: f1185a7567ce5119c4e675690fbe9d2b to self. 2024-11-21T01:46:13,286 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] handler.UnassignRegionHandler(157): Closed f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:13,286 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=159 updating hbase:meta row=f1185a7567ce5119c4e675690fbe9d2b, regionState=CLOSED 2024-11-21T01:46:13,287 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=160, ppid=159, state=RUNNABLE, hasLock=false; CloseRegionProcedure f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 because future has completed 2024-11-21T01:46:13,289 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=160, resume processing ppid=159 2024-11-21T01:46:13,289 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, ppid=159, state=SUCCESS, hasLock=false; CloseRegionProcedure f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 in 160 msec 2024-11-21T01:46:13,290 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=159, ppid=158, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f1185a7567ce5119c4e675690fbe9d2b, REOPEN/MOVE; state=CLOSED, location=b93ada835ef8,41129,1732153434359; forceNewPlan=false, retain=true 2024-11-21T01:46:13,440 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=159 updating hbase:meta row=f1185a7567ce5119c4e675690fbe9d2b, regionState=OPENING, regionLocation=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:13,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=159, ppid=158, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f1185a7567ce5119c4e675690fbe9d2b, REOPEN/MOVE because future has completed 2024-11-21T01:46:13,442 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=159, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359}] 2024-11-21T01:46:13,596 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:13,597 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(7752): Opening region: {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} 2024-11-21T01:46:13,597 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:13,597 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:46:13,597 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(7794): checking encryption for f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:13,597 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(7797): checking classloading for f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:13,598 INFO [StoreOpener-f1185a7567ce5119c4e675690fbe9d2b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:13,599 INFO [StoreOpener-f1185a7567ce5119c4e675690fbe9d2b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T01:46:13,599 INFO [StoreOpener-f1185a7567ce5119c4e675690fbe9d2b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1185a7567ce5119c4e675690fbe9d2b columnFamilyName A 2024-11-21T01:46:13,600 DEBUG [StoreOpener-f1185a7567ce5119c4e675690fbe9d2b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:13,600 INFO [StoreOpener-f1185a7567ce5119c4e675690fbe9d2b-1 {}] regionserver.HStore(327): Store=f1185a7567ce5119c4e675690fbe9d2b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:46:13,601 INFO [StoreOpener-f1185a7567ce5119c4e675690fbe9d2b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:13,601 INFO [StoreOpener-f1185a7567ce5119c4e675690fbe9d2b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T01:46:13,601 INFO [StoreOpener-f1185a7567ce5119c4e675690fbe9d2b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1185a7567ce5119c4e675690fbe9d2b columnFamilyName B 2024-11-21T01:46:13,601 DEBUG [StoreOpener-f1185a7567ce5119c4e675690fbe9d2b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:13,602 INFO [StoreOpener-f1185a7567ce5119c4e675690fbe9d2b-1 {}] regionserver.HStore(327): Store=f1185a7567ce5119c4e675690fbe9d2b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:46:13,602 INFO [StoreOpener-f1185a7567ce5119c4e675690fbe9d2b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:13,602 INFO [StoreOpener-f1185a7567ce5119c4e675690fbe9d2b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T01:46:13,602 INFO [StoreOpener-f1185a7567ce5119c4e675690fbe9d2b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1185a7567ce5119c4e675690fbe9d2b columnFamilyName C 2024-11-21T01:46:13,602 DEBUG [StoreOpener-f1185a7567ce5119c4e675690fbe9d2b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:13,603 INFO [StoreOpener-f1185a7567ce5119c4e675690fbe9d2b-1 {}] regionserver.HStore(327): Store=f1185a7567ce5119c4e675690fbe9d2b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:46:13,603 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:13,603 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(1038): replaying wal for f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:13,603 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:13,604 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:13,605 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(1048): stopping wal replay for f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:13,605 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(1060): Cleaning up temporary data for f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:13,606 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T01:46:13,607 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(1093): writing seq id for f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:13,607 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(1114): Opened f1185a7567ce5119c4e675690fbe9d2b; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60936429, jitterRate=-0.09197644889354706}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T01:46:13,607 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:13,608 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(1006): Region open journal for f1185a7567ce5119c4e675690fbe9d2b: Running coprocessor pre-open hook at 1732153573597Writing region info on filesystem at 1732153573597Initializing all the Stores at 1732153573598 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732153573598Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732153573598Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732153573598Cleaning up temporary data from old regions at 1732153573605 (+7 ms)Running coprocessor post-open hooks at 1732153573607 (+2 ms)Region opened successfully at 1732153573608 (+1 ms) 2024-11-21T01:46:13,609 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., pid=161, masterSystemTime=1732153573594 2024-11-21T01:46:13,610 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:13,610 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:13,611 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=159 updating hbase:meta row=f1185a7567ce5119c4e675690fbe9d2b, regionState=OPEN, openSeqNum=5, regionLocation=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:13,612 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=161, ppid=159, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 because future has completed 2024-11-21T01:46:13,614 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=159 2024-11-21T01:46:13,614 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=159, state=SUCCESS, hasLock=false; OpenRegionProcedure f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 in 171 msec 2024-11-21T01:46:13,615 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-21T01:46:13,616 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f1185a7567ce5119c4e675690fbe9d2b, REOPEN/MOVE in 490 msec 2024-11-21T01:46:13,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=158, resume processing ppid=157 2024-11-21T01:46:13,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, ppid=157, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 494 msec 2024-11-21T01:46:13,620 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 514 msec 2024-11-21T01:46:13,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=157 2024-11-21T01:46:13,621 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:46:13,621 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d756391, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,621 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:46:13,621 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:46:13,622 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:46:13,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:46:13,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:46:13,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ceb05bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:46:13,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:46:13,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:13,624 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:46:13,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5944ed1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,625 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30e225a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,625 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:46:13,625 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:46:13,625 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:46:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:46:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:46:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a6a9a9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:46:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:46:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:13,627 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:46:13,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61c3146f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,628 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fea92ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,628 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:46:13,628 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:46:13,628 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:46:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:46:13,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:46:13,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fc161b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:46:13,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:46:13,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:13,629 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34868, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:46:13,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@382a4f3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,630 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c187dca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,630 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:46:13,631 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:46:13,631 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:46:13,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:46:13,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:46:13,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27178fbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:46:13,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:46:13,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:13,632 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:46:13,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@225b21ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,633 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76ecb225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,633 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:46:13,633 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:46:13,634 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:46:13,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:46:13,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:46:13,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3df427b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:46:13,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:46:13,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:13,635 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:46:13,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11c83ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,636 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8b691ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,636 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:46:13,636 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:46:13,637 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:46:13,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:46:13,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:46:13,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@663e4f55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:46:13,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:46:13,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:13,638 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34932, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:46:13,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51bb07f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,639 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16d844d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,639 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:46:13,639 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:46:13,639 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:46:13,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:46:13,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:46:13,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64e60f7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:46:13,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:46:13,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:13,641 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:46:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76eba0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,642 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@533d9664, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,642 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:46:13,642 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:46:13,642 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:46:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:46:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:46:13,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d28012b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:46:13,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:46:13,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:13,643 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34976, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:46:13,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1419f76a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,645 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@383c5fef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,645 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:46:13,645 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:46:13,646 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:46:13,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:46:13,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:46:13,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34aa1bcd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:46:13,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:46:13,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:13,647 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34998, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:46:13,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@315325d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,648 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b279a63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,648 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,36607,-1 for getting cluster id 2024-11-21T01:46:13,648 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:46:13,648 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '809ac628-2316-4d75-a0fd-5a443f7d2388' 2024-11-21T01:46:13,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:46:13,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "809ac628-2316-4d75-a0fd-5a443f7d2388" 2024-11-21T01:46:13,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fedcc23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,36607,-1] 2024-11-21T01:46:13,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:46:13,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:13,649 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:46:13,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6687e27d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:46:13,650 DEBUG [Thread-2509 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:46:13,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:46:13,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:46:13,652 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:46:13,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5] 2024-11-21T01:46:13,653 DEBUG [Thread-2507 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:46:13,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:46:13,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:46:13,655 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48274, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:46:13,655 DEBUG [Thread-2511 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:46:13,655 DEBUG [Thread-2522 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:46:13,656 DEBUG [Thread-2524 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:46:13,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:46:13,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:46:13,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:46:13,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:46:13,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:46:13,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:46:13,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5] 2024-11-21T01:46:13,657 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48290, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:46:13,658 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:46:13,658 DEBUG [Thread-2515 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:46:13,658 DEBUG [Thread-2513 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:46:13,659 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48302, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:46:13,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:46:13,659 DEBUG [Thread-2515 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:46:13,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:46:13,660 DEBUG [Thread-2515 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:46:13,660 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48310, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:46:13,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5] 2024-11-21T01:46:13,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5] 2024-11-21T01:46:13,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5] 2024-11-21T01:46:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5] 2024-11-21T01:46:13,662 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48324, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:46:13,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5] 2024-11-21T01:46:13,666 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:46:13,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-21T01:46:13,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T01:46:13,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:13,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f1185a7567ce5119c4e675690fbe9d2b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T01:46:13,669 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:46:13,669 DEBUG [Thread-2518 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:46:13,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=A 2024-11-21T01:46:13,670 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:46:13,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:13,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=B 2024-11-21T01:46:13,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:13,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=C 2024-11-21T01:46:13,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:13,670 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:46:13,671 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:46:13,671 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:46:13,672 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48328, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:46:13,674 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5] 2024-11-21T01:46:13,675 DEBUG [Thread-2520 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:46:13,678 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:46:13,678 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:46:13,681 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48332, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:46:13,681 DEBUG [Thread-2526 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:46:13,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,41129,1732153434359, seqNum=-1] 2024-11-21T01:46:13,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:46:13,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5] 2024-11-21T01:46:13,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:46:13,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5] 2024-11-21T01:46:13,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:13,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153633692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:13,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:13,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:13,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:13,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:13,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153633694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:13,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:13,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:13,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:13,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:13,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153633694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:13,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:13,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:13,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:13,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:13,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153633697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:13,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:13,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:13,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:13,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:13,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153633700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:13,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:13,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:13,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:13,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214c0a645b67bf40c291aff843bb916dac_f1185a7567ce5119c4e675690fbe9d2b is 50, key is test_row_0/A:col10/1732153573666/Put/seqid=0 2024-11-21T01:46:13,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742392_1568 (size=9714) 2024-11-21T01:46:13,770 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:13,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T01:46:13,775 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214c0a645b67bf40c291aff843bb916dac_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214c0a645b67bf40c291aff843bb916dac_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:13,776 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/c1fa2a8a785445669e72f02e9581f560, store: [table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:13,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/c1fa2a8a785445669e72f02e9581f560 is 175, key is test_row_0/A:col10/1732153573666/Put/seqid=0 2024-11-21T01:46:13,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742393_1569 (size=22361) 2024-11-21T01:46:13,804 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/c1fa2a8a785445669e72f02e9581f560 2024-11-21T01:46:13,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:13,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153633803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:13,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:13,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153633803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:13,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:13,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:13,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153633803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:13,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153633803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:13,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:13,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:13,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:13,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:13,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:13,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153633813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:13,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:13,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:13,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:13,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-21T01:46:13,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:13,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:13,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:13,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:13,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:13,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:13,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/0ff599c220c04b49adbd3a10ef355815 is 50, key is test_row_0/B:col10/1732153573666/Put/seqid=0 2024-11-21T01:46:13,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742394_1570 (size=9657) 2024-11-21T01:46:13,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-21T01:46:13,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:13,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:13,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:13,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:13,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:13,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:13,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T01:46:14,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:14,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153634012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:14,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:14,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153634013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:14,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:14,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153634013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:14,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:14,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:14,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:14,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153634013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:14,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:14,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:14,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:14,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153634023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:14,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:14,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-21T01:46:14,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:14,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:14,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:14,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:14,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:14,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:14,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/0ff599c220c04b49adbd3a10ef355815 2024-11-21T01:46:14,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-21T01:46:14,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:14,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:14,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:14,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:14,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:14,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:14,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T01:46:14,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/ea3d9c1d94c74620b2ca662e0fb0c3e8 is 50, key is test_row_0/C:col10/1732153573666/Put/seqid=0 2024-11-21T01:46:14,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:14,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153634323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153634323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:14,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:14,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:14,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:14,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153634323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:14,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153634325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:14,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:14,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:14,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:14,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153634332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:14,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:14,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742395_1571 (size=9657) 2024-11-21T01:46:14,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/ea3d9c1d94c74620b2ca662e0fb0c3e8 2024-11-21T01:46:14,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/c1fa2a8a785445669e72f02e9581f560 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/c1fa2a8a785445669e72f02e9581f560 2024-11-21T01:46:14,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/c1fa2a8a785445669e72f02e9581f560, entries=100, sequenceid=16, filesize=21.8 K 2024-11-21T01:46:14,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/0ff599c220c04b49adbd3a10ef355815 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/0ff599c220c04b49adbd3a10ef355815 2024-11-21T01:46:14,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/0ff599c220c04b49adbd3a10ef355815, entries=100, sequenceid=16, filesize=9.4 K 2024-11-21T01:46:14,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/ea3d9c1d94c74620b2ca662e0fb0c3e8 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/ea3d9c1d94c74620b2ca662e0fb0c3e8 2024-11-21T01:46:14,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/ea3d9c1d94c74620b2ca662e0fb0c3e8, entries=100, sequenceid=16, filesize=9.4 K 2024-11-21T01:46:14,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f1185a7567ce5119c4e675690fbe9d2b in 710ms, sequenceid=16, compaction requested=false 2024-11-21T01:46:14,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:14,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-21T01:46:14,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:14,446 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing f1185a7567ce5119c4e675690fbe9d2b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T01:46:14,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=A 2024-11-21T01:46:14,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:14,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=B 2024-11-21T01:46:14,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:14,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=C 2024-11-21T01:46:14,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:14,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ecc627d030cc426b8a4058f4798036cc_f1185a7567ce5119c4e675690fbe9d2b is 50, key is test_row_0/A:col10/1732153573694/Put/seqid=0 2024-11-21T01:46:14,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742396_1572 (size=12154) 2024-11-21T01:46:14,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:14,494 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ecc627d030cc426b8a4058f4798036cc_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ecc627d030cc426b8a4058f4798036cc_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:14,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/bfb44d5463ed45168fd38576a37f13c8, store: [table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:14,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/bfb44d5463ed45168fd38576a37f13c8 is 175, key is test_row_0/A:col10/1732153573694/Put/seqid=0 2024-11-21T01:46:14,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742397_1573 (size=30955) 2024-11-21T01:46:14,519 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/bfb44d5463ed45168fd38576a37f13c8 2024-11-21T01:46:14,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/08de650fbfdc4e408ea92fc821fb4908 is 50, key is test_row_0/B:col10/1732153573694/Put/seqid=0 2024-11-21T01:46:14,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742398_1574 (size=12001) 2024-11-21T01:46:14,564 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/08de650fbfdc4e408ea92fc821fb4908 2024-11-21T01:46:14,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/106a718c347a4aa49bcc28469e28c383 is 50, key is test_row_0/C:col10/1732153573694/Put/seqid=0 2024-11-21T01:46:14,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742399_1575 (size=12001) 2024-11-21T01:46:14,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T01:46:14,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:14,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:14,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:14,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153634848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:14,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:14,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:14,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153634849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:14,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:14,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153634852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:14,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:14,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:14,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153634853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:14,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:14,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:14,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153634854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:14,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:14,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:14,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153634963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:14,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153634963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:14,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:14,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153634963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:14,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153634963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:14,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:14,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153634963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:14,992 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/106a718c347a4aa49bcc28469e28c383 2024-11-21T01:46:14,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/bfb44d5463ed45168fd38576a37f13c8 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/bfb44d5463ed45168fd38576a37f13c8 2024-11-21T01:46:15,006 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/bfb44d5463ed45168fd38576a37f13c8, entries=150, sequenceid=36, filesize=30.2 K 2024-11-21T01:46:15,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/08de650fbfdc4e408ea92fc821fb4908 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/08de650fbfdc4e408ea92fc821fb4908 2024-11-21T01:46:15,011 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/08de650fbfdc4e408ea92fc821fb4908, entries=150, sequenceid=36, filesize=11.7 K 2024-11-21T01:46:15,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/106a718c347a4aa49bcc28469e28c383 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/106a718c347a4aa49bcc28469e28c383 2024-11-21T01:46:15,015 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/106a718c347a4aa49bcc28469e28c383, entries=150, sequenceid=36, filesize=11.7 K 2024-11-21T01:46:15,016 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for f1185a7567ce5119c4e675690fbe9d2b in 570ms, sequenceid=36, compaction requested=false 2024-11-21T01:46:15,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:15,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:15,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-21T01:46:15,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-21T01:46:15,020 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-21T01:46:15,020 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3470 sec 2024-11-21T01:46:15,022 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.3550 sec 2024-11-21T01:46:15,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:15,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f1185a7567ce5119c4e675690fbe9d2b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T01:46:15,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=A 2024-11-21T01:46:15,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:15,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=B 2024-11-21T01:46:15,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:15,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=C 2024-11-21T01:46:15,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:15,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f27aecd2005e424190257b80c8f65d91_f1185a7567ce5119c4e675690fbe9d2b is 50, key is test_row_0/A:col10/1732153575175/Put/seqid=0 2024-11-21T01:46:15,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153635203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:15,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:15,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:15,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153635207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:15,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:15,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153635207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:15,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:15,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:15,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153635209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:15,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:15,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153635210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:15,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:15,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742400_1576 (size=17034) 2024-11-21T01:46:15,230 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:15,250 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f27aecd2005e424190257b80c8f65d91_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f27aecd2005e424190257b80c8f65d91_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:15,258 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/807e33901df04b5794f47f2db3fe554f, store: [table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:15,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/807e33901df04b5794f47f2db3fe554f is 175, key is test_row_0/A:col10/1732153575175/Put/seqid=0 2024-11-21T01:46:15,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742401_1577 (size=48139) 2024-11-21T01:46:15,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:15,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153635317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:15,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:15,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:15,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153635330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:15,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:15,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153635330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:15,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:15,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:15,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:15,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153635333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:15,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:15,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153635337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:15,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:15,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:15,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153635533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:15,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:15,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:15,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153635543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:15,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:15,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:15,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153635543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:15,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:15,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:15,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153635543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:15,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:15,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:15,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153635553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:15,706 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/807e33901df04b5794f47f2db3fe554f 2024-11-21T01:46:15,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/c4c0192df5244a4ca6ff96919ed79625 is 50, key is test_row_0/B:col10/1732153575175/Put/seqid=0 2024-11-21T01:46:15,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742402_1578 (size=12001) 2024-11-21T01:46:15,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T01:46:15,823 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:46:15,823 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:46:15,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-21T01:46:15,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T01:46:15,826 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:46:15,827 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:46:15,827 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:46:15,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:15,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153635843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:15,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:15,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153635853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:15,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:15,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153635853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:15,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:15,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:15,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153635854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:15,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:15,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:15,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153635863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:15,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:15,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:15,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T01:46:15,971 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T01:46:15,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T01:46:15,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:15,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:15,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:15,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:15,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:15,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:16,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/c4c0192df5244a4ca6ff96919ed79625 2024-11-21T01:46:16,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T01:46:16,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:16,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:16,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:16,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:16,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:16,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:16,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T01:46:16,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/7b97a41ace234bfca5ed75599b6fda82 is 50, key is test_row_0/C:col10/1732153575175/Put/seqid=0 2024-11-21T01:46:16,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742403_1579 (size=12001) 2024-11-21T01:46:16,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/7b97a41ace234bfca5ed75599b6fda82 2024-11-21T01:46:16,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/807e33901df04b5794f47f2db3fe554f as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/807e33901df04b5794f47f2db3fe554f 2024-11-21T01:46:16,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/807e33901df04b5794f47f2db3fe554f, entries=250, sequenceid=52, filesize=47.0 K 2024-11-21T01:46:16,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/c4c0192df5244a4ca6ff96919ed79625 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/c4c0192df5244a4ca6ff96919ed79625 2024-11-21T01:46:16,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/c4c0192df5244a4ca6ff96919ed79625, entries=150, sequenceid=52, filesize=11.7 K 2024-11-21T01:46:16,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/7b97a41ace234bfca5ed75599b6fda82 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/7b97a41ace234bfca5ed75599b6fda82 2024-11-21T01:46:16,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/7b97a41ace234bfca5ed75599b6fda82, entries=150, sequenceid=52, filesize=11.7 K 2024-11-21T01:46:16,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for f1185a7567ce5119c4e675690fbe9d2b in 1025ms, sequenceid=52, compaction requested=true 2024-11-21T01:46:16,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:16,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f1185a7567ce5119c4e675690fbe9d2b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:46:16,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:16,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f1185a7567ce5119c4e675690fbe9d2b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:46:16,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:16,203 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:46:16,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f1185a7567ce5119c4e675690fbe9d2b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:46:16,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:46:16,203 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:46:16,204 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:46:16,204 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): f1185a7567ce5119c4e675690fbe9d2b/B is initiating minor compaction (all files) 2024-11-21T01:46:16,204 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f1185a7567ce5119c4e675690fbe9d2b/B in TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:16,204 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/0ff599c220c04b49adbd3a10ef355815, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/08de650fbfdc4e408ea92fc821fb4908, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/c4c0192df5244a4ca6ff96919ed79625] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp, totalSize=32.9 K 2024-11-21T01:46:16,205 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101455 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:46:16,205 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): f1185a7567ce5119c4e675690fbe9d2b/A is initiating minor compaction (all files) 2024-11-21T01:46:16,205 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f1185a7567ce5119c4e675690fbe9d2b/A in TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:16,205 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/c1fa2a8a785445669e72f02e9581f560, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/bfb44d5463ed45168fd38576a37f13c8, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/807e33901df04b5794f47f2db3fe554f] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp, totalSize=99.1 K 2024-11-21T01:46:16,205 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:16,205 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. files: [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/c1fa2a8a785445669e72f02e9581f560, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/bfb44d5463ed45168fd38576a37f13c8, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/807e33901df04b5794f47f2db3fe554f] 2024-11-21T01:46:16,206 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ff599c220c04b49adbd3a10ef355815, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732153573666 2024-11-21T01:46:16,206 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1fa2a8a785445669e72f02e9581f560, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732153573666 2024-11-21T01:46:16,206 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 08de650fbfdc4e408ea92fc821fb4908, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732153573690 2024-11-21T01:46:16,206 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfb44d5463ed45168fd38576a37f13c8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732153573690 2024-11-21T01:46:16,207 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting c4c0192df5244a4ca6ff96919ed79625, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732153575175 2024-11-21T01:46:16,207 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 807e33901df04b5794f47f2db3fe554f, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732153574849 2024-11-21T01:46:16,216 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f1185a7567ce5119c4e675690fbe9d2b#B#compaction#498 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:16,217 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/73db08aa8bc04b37bd50ec11eb19c6dc is 50, key is test_row_0/B:col10/1732153575175/Put/seqid=0 2024-11-21T01:46:16,222 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:16,226 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411216c1c77e3795d497e8a7aa4d126a66596_f1185a7567ce5119c4e675690fbe9d2b store=[table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:16,228 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411216c1c77e3795d497e8a7aa4d126a66596_f1185a7567ce5119c4e675690fbe9d2b, store=[table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:16,229 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216c1c77e3795d497e8a7aa4d126a66596_f1185a7567ce5119c4e675690fbe9d2b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:16,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742404_1580 (size=12104) 2024-11-21T01:46:16,253 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/73db08aa8bc04b37bd50ec11eb19c6dc as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/73db08aa8bc04b37bd50ec11eb19c6dc 2024-11-21T01:46:16,258 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f1185a7567ce5119c4e675690fbe9d2b/B of f1185a7567ce5119c4e675690fbe9d2b into 73db08aa8bc04b37bd50ec11eb19c6dc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:16,258 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:16,258 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., storeName=f1185a7567ce5119c4e675690fbe9d2b/B, priority=13, startTime=1732153576203; duration=0sec 2024-11-21T01:46:16,258 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:46:16,258 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f1185a7567ce5119c4e675690fbe9d2b:B 2024-11-21T01:46:16,258 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:46:16,259 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:46:16,259 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): f1185a7567ce5119c4e675690fbe9d2b/C is initiating minor compaction (all files) 2024-11-21T01:46:16,260 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f1185a7567ce5119c4e675690fbe9d2b/C in TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:16,261 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/ea3d9c1d94c74620b2ca662e0fb0c3e8, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/106a718c347a4aa49bcc28469e28c383, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/7b97a41ace234bfca5ed75599b6fda82] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp, totalSize=32.9 K 2024-11-21T01:46:16,261 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting ea3d9c1d94c74620b2ca662e0fb0c3e8, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732153573666 2024-11-21T01:46:16,262 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 106a718c347a4aa49bcc28469e28c383, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732153573690 2024-11-21T01:46:16,262 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b97a41ace234bfca5ed75599b6fda82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732153575175 2024-11-21T01:46:16,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742405_1581 (size=4469) 2024-11-21T01:46:16,279 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f1185a7567ce5119c4e675690fbe9d2b#C#compaction#500 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:16,280 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/f4737e475a6649c69824655e4accb6f5 is 50, key is test_row_0/C:col10/1732153575175/Put/seqid=0 2024-11-21T01:46:16,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T01:46:16,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:16,292 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing f1185a7567ce5119c4e675690fbe9d2b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-21T01:46:16,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=A 2024-11-21T01:46:16,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:16,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=B 2024-11-21T01:46:16,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:16,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=C 2024-11-21T01:46:16,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:16,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742406_1582 (size=12104) 2024-11-21T01:46:16,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121de06fd8c936d45a394193e77c98e6e58_f1185a7567ce5119c4e675690fbe9d2b is 50, key is test_row_0/A:col10/1732153575206/Put/seqid=0 2024-11-21T01:46:16,304 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/f4737e475a6649c69824655e4accb6f5 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/f4737e475a6649c69824655e4accb6f5 2024-11-21T01:46:16,310 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f1185a7567ce5119c4e675690fbe9d2b/C of f1185a7567ce5119c4e675690fbe9d2b into f4737e475a6649c69824655e4accb6f5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:16,310 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:16,310 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., storeName=f1185a7567ce5119c4e675690fbe9d2b/C, priority=13, startTime=1732153576203; duration=0sec 2024-11-21T01:46:16,310 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:16,310 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f1185a7567ce5119c4e675690fbe9d2b:C 2024-11-21T01:46:16,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742407_1583 (size=12154) 2024-11-21T01:46:16,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:16,318 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121de06fd8c936d45a394193e77c98e6e58_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121de06fd8c936d45a394193e77c98e6e58_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:16,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/55a6b6dbdc484698ae5800ec0dca1dda, store: [table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:16,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/55a6b6dbdc484698ae5800ec0dca1dda is 175, key is test_row_0/A:col10/1732153575206/Put/seqid=0 2024-11-21T01:46:16,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742408_1584 (size=30955) 2024-11-21T01:46:16,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:16,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:16,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:16,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153636377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:16,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:16,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:16,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:16,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:16,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153636377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:16,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:16,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:16,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:16,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153636378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:16,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:16,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:16,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:16,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153636380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:16,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:16,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153636383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:16,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:16,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:16,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:16,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T01:46:16,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:16,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153636492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:16,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:16,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:16,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:16,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153636493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:16,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:16,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:16,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:16,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:16,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153636493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:16,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:16,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:16,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:16,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:16,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153636493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:16,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:16,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:16,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:16,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:16,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:16,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153636493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:16,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:16,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:16,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:16,670 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f1185a7567ce5119c4e675690fbe9d2b#A#compaction#499 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:16,671 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/2cf48b36feca49d1a5d69212cc5d78e8 is 175, key is test_row_0/A:col10/1732153575175/Put/seqid=0 2024-11-21T01:46:16,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742409_1585 (size=31058) 2024-11-21T01:46:16,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:16,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153636702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:16,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:16,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153636702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:16,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:16,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:16,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:16,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:16,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153636703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:16,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:16,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:16,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:16,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153636703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:16,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:16,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:16,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:16,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:16,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:16,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153636703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:16,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:16,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:16,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:16,728 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=69, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/55a6b6dbdc484698ae5800ec0dca1dda 2024-11-21T01:46:16,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/70ab50c871bf49619554d723ce72842b is 50, key is test_row_0/B:col10/1732153575206/Put/seqid=0 2024-11-21T01:46:16,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742410_1586 (size=12001) 2024-11-21T01:46:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T01:46:17,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153637013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:17,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:17,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153637013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:17,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:17,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:17,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153637013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:17,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153637014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:17,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153637014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:17,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:17,090 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/2cf48b36feca49d1a5d69212cc5d78e8 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/2cf48b36feca49d1a5d69212cc5d78e8 2024-11-21T01:46:17,094 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f1185a7567ce5119c4e675690fbe9d2b/A of f1185a7567ce5119c4e675690fbe9d2b into 2cf48b36feca49d1a5d69212cc5d78e8(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:17,094 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:17,094 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., storeName=f1185a7567ce5119c4e675690fbe9d2b/A, priority=13, startTime=1732153576203; duration=0sec 2024-11-21T01:46:17,094 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:17,094 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f1185a7567ce5119c4e675690fbe9d2b:A 2024-11-21T01:46:17,182 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/70ab50c871bf49619554d723ce72842b 2024-11-21T01:46:17,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/0bba499e048f41d7885bca7626177462 is 50, key is test_row_0/C:col10/1732153575206/Put/seqid=0 2024-11-21T01:46:17,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742411_1587 (size=12001) 2024-11-21T01:46:17,192 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/0bba499e048f41d7885bca7626177462 2024-11-21T01:46:17,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/55a6b6dbdc484698ae5800ec0dca1dda as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/55a6b6dbdc484698ae5800ec0dca1dda 2024-11-21T01:46:17,200 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/55a6b6dbdc484698ae5800ec0dca1dda, entries=150, sequenceid=69, filesize=30.2 K 2024-11-21T01:46:17,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/70ab50c871bf49619554d723ce72842b as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/70ab50c871bf49619554d723ce72842b 2024-11-21T01:46:17,204 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/70ab50c871bf49619554d723ce72842b, entries=150, sequenceid=69, filesize=11.7 K 2024-11-21T01:46:17,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/0bba499e048f41d7885bca7626177462 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/0bba499e048f41d7885bca7626177462 2024-11-21T01:46:17,208 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/0bba499e048f41d7885bca7626177462, entries=150, sequenceid=69, filesize=11.7 K 2024-11-21T01:46:17,209 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for f1185a7567ce5119c4e675690fbe9d2b in 917ms, sequenceid=69, compaction requested=false 2024-11-21T01:46:17,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:17,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:17,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-21T01:46:17,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-21T01:46:17,214 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-21T01:46:17,214 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3840 sec 2024-11-21T01:46:17,216 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.3920 sec 2024-11-21T01:46:17,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:17,527 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f1185a7567ce5119c4e675690fbe9d2b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-21T01:46:17,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=A 2024-11-21T01:46:17,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:17,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=B 2024-11-21T01:46:17,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:17,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=C 2024-11-21T01:46:17,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:17,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411217ad810caf4754566af3c59295d9f02b7_f1185a7567ce5119c4e675690fbe9d2b is 50, key is test_row_0/A:col10/1732153577526/Put/seqid=0 2024-11-21T01:46:17,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742412_1588 (size=12154) 2024-11-21T01:46:17,544 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:17,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:17,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153637537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:17,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:17,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:17,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153637541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:17,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:17,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153637542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:17,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:17,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:17,548 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411217ad810caf4754566af3c59295d9f02b7_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217ad810caf4754566af3c59295d9f02b7_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:17,549 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/1a4f602decfc47bb992827c7435e5eac, store: [table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:17,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/1a4f602decfc47bb992827c7435e5eac is 175, key is test_row_0/A:col10/1732153577526/Put/seqid=0 2024-11-21T01:46:17,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742413_1589 (size=30955) 2024-11-21T01:46:17,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:17,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153637544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:17,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:17,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:17,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153637546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:17,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:17,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:17,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153637653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:17,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:17,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153637653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:17,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:17,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:17,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153637653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:17,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:17,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:17,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:17,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153637662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:17,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:17,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:17,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153637663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:17,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:17,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:17,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153637863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:17,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153637863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:17,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:17,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153637873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:17,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:17,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153637873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:17,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:17,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:17,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153637873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:17,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:17,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:17,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:17,963 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=90, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/1a4f602decfc47bb992827c7435e5eac 2024-11-21T01:46:17,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T01:46:17,973 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:46:17,974 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:46:17,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-21T01:46:17,976 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:46:17,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T01:46:17,977 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:46:17,977 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:46:17,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/5fc8aa0965054a8ca6d74add971a518c is 50, key is test_row_0/B:col10/1732153577526/Put/seqid=0 2024-11-21T01:46:18,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742414_1590 (size=12001) 2024-11-21T01:46:18,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T01:46:18,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T01:46:18,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:18,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:18,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:18,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:18,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:18,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:18,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153638179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:18,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:18,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153638179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:18,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:18,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:18,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:18,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:18,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:18,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:18,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:18,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153638183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:18,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:18,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:18,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:18,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:18,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153638183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:18,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:18,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:18,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:18,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:18,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153638183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:18,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:18,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:18,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:18,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T01:46:18,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:18,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:18,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:18,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:18,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:18,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T01:46:18,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/5fc8aa0965054a8ca6d74add971a518c 2024-11-21T01:46:18,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T01:46:18,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:18,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:18,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:18,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:18,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:18,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:18,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/ef66995a386e480ea8425ac8eb1d6ac8 is 50, key is test_row_0/C:col10/1732153577526/Put/seqid=0 2024-11-21T01:46:18,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742415_1591 (size=12001) 2024-11-21T01:46:18,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/ef66995a386e480ea8425ac8eb1d6ac8 2024-11-21T01:46:18,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/1a4f602decfc47bb992827c7435e5eac as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/1a4f602decfc47bb992827c7435e5eac 2024-11-21T01:46:18,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/1a4f602decfc47bb992827c7435e5eac, entries=150, sequenceid=90, filesize=30.2 K 2024-11-21T01:46:18,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/5fc8aa0965054a8ca6d74add971a518c as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/5fc8aa0965054a8ca6d74add971a518c 2024-11-21T01:46:18,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/5fc8aa0965054a8ca6d74add971a518c, entries=150, sequenceid=90, filesize=11.7 K 2024-11-21T01:46:18,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/ef66995a386e480ea8425ac8eb1d6ac8 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/ef66995a386e480ea8425ac8eb1d6ac8 2024-11-21T01:46:18,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/ef66995a386e480ea8425ac8eb1d6ac8, entries=150, sequenceid=90, filesize=11.7 K 2024-11-21T01:46:18,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f1185a7567ce5119c4e675690fbe9d2b in 1053ms, sequenceid=90, compaction requested=true 2024-11-21T01:46:18,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:18,580 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:46:18,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f1185a7567ce5119c4e675690fbe9d2b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:46:18,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:18,580 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:46:18,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f1185a7567ce5119c4e675690fbe9d2b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:46:18,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:18,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f1185a7567ce5119c4e675690fbe9d2b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:46:18,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:46:18,581 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:46:18,581 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): f1185a7567ce5119c4e675690fbe9d2b/A is initiating minor compaction (all files) 2024-11-21T01:46:18,581 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f1185a7567ce5119c4e675690fbe9d2b/A in TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:18,581 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/2cf48b36feca49d1a5d69212cc5d78e8, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/55a6b6dbdc484698ae5800ec0dca1dda, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/1a4f602decfc47bb992827c7435e5eac] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp, totalSize=90.8 K 2024-11-21T01:46:18,581 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:18,582 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. files: [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/2cf48b36feca49d1a5d69212cc5d78e8, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/55a6b6dbdc484698ae5800ec0dca1dda, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/1a4f602decfc47bb992827c7435e5eac] 2024-11-21T01:46:18,582 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cf48b36feca49d1a5d69212cc5d78e8, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732153575175 2024-11-21T01:46:18,582 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:46:18,582 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): f1185a7567ce5119c4e675690fbe9d2b/B is initiating minor compaction (all files) 2024-11-21T01:46:18,582 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f1185a7567ce5119c4e675690fbe9d2b/B in TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:18,583 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/73db08aa8bc04b37bd50ec11eb19c6dc, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/70ab50c871bf49619554d723ce72842b, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/5fc8aa0965054a8ca6d74add971a518c] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp, totalSize=35.3 K 2024-11-21T01:46:18,583 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55a6b6dbdc484698ae5800ec0dca1dda, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732153575205 2024-11-21T01:46:18,585 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a4f602decfc47bb992827c7435e5eac, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732153577525 2024-11-21T01:46:18,585 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 73db08aa8bc04b37bd50ec11eb19c6dc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732153575175 2024-11-21T01:46:18,586 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 70ab50c871bf49619554d723ce72842b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732153575205 2024-11-21T01:46:18,586 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fc8aa0965054a8ca6d74add971a518c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732153577525 2024-11-21T01:46:18,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T01:46:18,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:18,592 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing f1185a7567ce5119c4e675690fbe9d2b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T01:46:18,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=A 2024-11-21T01:46:18,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:18,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=B 2024-11-21T01:46:18,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:18,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=C 2024-11-21T01:46:18,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:18,598 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:18,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T01:46:18,611 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f1185a7567ce5119c4e675690fbe9d2b#B#compaction#508 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:18,620 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/234a6f0c1fb843388278d844dd42726d is 50, key is test_row_0/B:col10/1732153577526/Put/seqid=0 2024-11-21T01:46:18,621 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121d0c8a7710692427197e368b4f4ce93ce_f1185a7567ce5119c4e675690fbe9d2b store=[table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:18,623 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121d0c8a7710692427197e368b4f4ce93ce_f1185a7567ce5119c4e675690fbe9d2b, store=[table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:18,623 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d0c8a7710692427197e368b4f4ce93ce_f1185a7567ce5119c4e675690fbe9d2b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:18,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112108665353da6746d5b3effac7546d7bd0_f1185a7567ce5119c4e675690fbe9d2b is 50, key is test_row_0/A:col10/1732153577542/Put/seqid=0 2024-11-21T01:46:18,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742416_1592 (size=12207) 2024-11-21T01:46:18,646 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/234a6f0c1fb843388278d844dd42726d as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/234a6f0c1fb843388278d844dd42726d 2024-11-21T01:46:18,651 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f1185a7567ce5119c4e675690fbe9d2b/B of f1185a7567ce5119c4e675690fbe9d2b into 234a6f0c1fb843388278d844dd42726d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:18,651 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:18,651 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., storeName=f1185a7567ce5119c4e675690fbe9d2b/B, priority=13, startTime=1732153578580; duration=0sec 2024-11-21T01:46:18,651 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:46:18,651 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f1185a7567ce5119c4e675690fbe9d2b:B 2024-11-21T01:46:18,651 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:46:18,652 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:46:18,652 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): f1185a7567ce5119c4e675690fbe9d2b/C is initiating minor compaction (all files) 2024-11-21T01:46:18,652 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f1185a7567ce5119c4e675690fbe9d2b/C in TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:18,653 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/f4737e475a6649c69824655e4accb6f5, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/0bba499e048f41d7885bca7626177462, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/ef66995a386e480ea8425ac8eb1d6ac8] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp, totalSize=35.3 K 2024-11-21T01:46:18,667 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting f4737e475a6649c69824655e4accb6f5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732153575175 2024-11-21T01:46:18,669 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bba499e048f41d7885bca7626177462, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732153575205 2024-11-21T01:46:18,669 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting ef66995a386e480ea8425ac8eb1d6ac8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732153577525 2024-11-21T01:46:18,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742417_1593 (size=4469) 2024-11-21T01:46:18,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742418_1594 (size=12154) 2024-11-21T01:46:18,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:18,671 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f1185a7567ce5119c4e675690fbe9d2b#A#compaction#507 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:18,672 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/c7fb1619a98643138da84776082d5de3 is 175, key is test_row_0/A:col10/1732153577526/Put/seqid=0 2024-11-21T01:46:18,678 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112108665353da6746d5b3effac7546d7bd0_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112108665353da6746d5b3effac7546d7bd0_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:18,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/b7680dcff0e445c9b96f1288b552c993, store: [table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:18,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/b7680dcff0e445c9b96f1288b552c993 is 175, key is test_row_0/A:col10/1732153577542/Put/seqid=0 2024-11-21T01:46:18,687 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f1185a7567ce5119c4e675690fbe9d2b#C#compaction#510 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:18,687 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/34e19aec69894be687734e9e46e0d6e0 is 50, key is test_row_0/C:col10/1732153577526/Put/seqid=0 2024-11-21T01:46:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:18,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:18,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742419_1595 (size=31161) 2024-11-21T01:46:18,709 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/c7fb1619a98643138da84776082d5de3 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/c7fb1619a98643138da84776082d5de3 2024-11-21T01:46:18,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742420_1596 (size=30955) 2024-11-21T01:46:18,714 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=103, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/b7680dcff0e445c9b96f1288b552c993 2024-11-21T01:46:18,715 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f1185a7567ce5119c4e675690fbe9d2b/A of f1185a7567ce5119c4e675690fbe9d2b into c7fb1619a98643138da84776082d5de3(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:18,715 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:18,715 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., storeName=f1185a7567ce5119c4e675690fbe9d2b/A, priority=13, startTime=1732153578580; duration=0sec 2024-11-21T01:46:18,715 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:18,715 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f1185a7567ce5119c4e675690fbe9d2b:A 2024-11-21T01:46:18,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/317de57286894d3dac7dd2eca9233857 is 50, key is test_row_0/B:col10/1732153577542/Put/seqid=0 2024-11-21T01:46:18,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742421_1597 (size=12207) 2024-11-21T01:46:18,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742422_1598 (size=12001) 2024-11-21T01:46:18,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732153578712, max timestamp=1732153578725Num uniques 859; 2024-11-21T01:46:18,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T01:46:18,732 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/34e19aec69894be687734e9e46e0d6e0 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/34e19aec69894be687734e9e46e0d6e0 2024-11-21T01:46:18,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732153578712, max timestamp=1732153578725Num uniques 859; 2024-11-21T01:46:18,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T01:46:18,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732153578714, max timestamp=1732153578725Num uniques 859; 2024-11-21T01:46:18,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T01:46:18,742 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f1185a7567ce5119c4e675690fbe9d2b/C of f1185a7567ce5119c4e675690fbe9d2b into 34e19aec69894be687734e9e46e0d6e0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:18,742 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:18,742 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., storeName=f1185a7567ce5119c4e675690fbe9d2b/C, priority=13, startTime=1732153578581; duration=0sec 2024-11-21T01:46:18,742 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:18,742 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f1185a7567ce5119c4e675690fbe9d2b:C 2024-11-21T01:46:18,749 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/317de57286894d3dac7dd2eca9233857 2024-11-21T01:46:18,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/374fa21782d24f4bb3f69e59cb5e1366 is 50, key is test_row_0/C:col10/1732153577542/Put/seqid=0 2024-11-21T01:46:18,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153638752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:18,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153638762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:18,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153638762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:18,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:18,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:18,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153638763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:18,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:18,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:18,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:18,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153638764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:18,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:18,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:18,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:18,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:18,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:18,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:18,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:18,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742423_1599 (size=12001) 2024-11-21T01:46:18,785 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/374fa21782d24f4bb3f69e59cb5e1366 2024-11-21T01:46:18,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/b7680dcff0e445c9b96f1288b552c993 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/b7680dcff0e445c9b96f1288b552c993 2024-11-21T01:46:18,795 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/b7680dcff0e445c9b96f1288b552c993, entries=150, sequenceid=103, filesize=30.2 K 2024-11-21T01:46:18,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/317de57286894d3dac7dd2eca9233857 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/317de57286894d3dac7dd2eca9233857 2024-11-21T01:46:18,799 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/317de57286894d3dac7dd2eca9233857, entries=150, sequenceid=103, filesize=11.7 K 2024-11-21T01:46:18,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/374fa21782d24f4bb3f69e59cb5e1366 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/374fa21782d24f4bb3f69e59cb5e1366 2024-11-21T01:46:18,804 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/374fa21782d24f4bb3f69e59cb5e1366, entries=150, sequenceid=103, filesize=11.7 K 2024-11-21T01:46:18,810 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=118.53 KB/121370 for f1185a7567ce5119c4e675690fbe9d2b in 218ms, sequenceid=103, compaction requested=false 2024-11-21T01:46:18,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:18,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:18,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-21T01:46:18,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-21T01:46:18,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-21T01:46:18,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 835 msec 2024-11-21T01:46:18,817 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 841 msec 2024-11-21T01:46:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:18,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f1185a7567ce5119c4e675690fbe9d2b 3/3 column families, dataSize=125.23 KB heapSize=355.30 KB 2024-11-21T01:46:18,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=A 2024-11-21T01:46:18,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:46:18,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=B 2024-11-21T01:46:18,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:46:18,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=C 2024-11-21T01:46:18,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:46:18,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ef713752a2824010a0a78160d596db50_f1185a7567ce5119c4e675690fbe9d2b is 50, key is test_row_0/A:col10/1732153578761/Put/seqid=0 2024-11-21T01:46:18,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153638882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:18,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:18,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:18,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:18,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153638889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:18,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153638889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:18,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:18,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:18,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:18,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:18,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:18,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:18,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153638892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:18,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153638892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:18,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742424_1600 (size=12304) 2024-11-21T01:46:18,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153638993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:18,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:18,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:18,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:19,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153639003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:19,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:19,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:19,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153639003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:19,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:19,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:19,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:19,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153639003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:19,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:19,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:19,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153639003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:19,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:19,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:19,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:19,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:19,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:19,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T01:46:19,113 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:46:19,113 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:46:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-21T01:46:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-21T01:46:19,115 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:46:19,116 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:46:19,116 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:46:19,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:19,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153639203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:19,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:19,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:19,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:19,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153639214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:19,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153639214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:19,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:19,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:19,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:19,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153639214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:19,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:19,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:19,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:19,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:19,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:19,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:19,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:19,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153639214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:19,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:19,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:19,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:19,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-21T01:46:19,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-21T01:46:19,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:19,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:19,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:19,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:19,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:19,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:19,327 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:19,332 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ef713752a2824010a0a78160d596db50_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ef713752a2824010a0a78160d596db50_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:19,333 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/29b46d3d15794e3a9035fcd0388e5d32, store: [table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:19,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/29b46d3d15794e3a9035fcd0388e5d32 is 175, key is test_row_0/A:col10/1732153578761/Put/seqid=0 2024-11-21T01:46:19,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742425_1601 (size=31105) 2024-11-21T01:46:19,341 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=141, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/29b46d3d15794e3a9035fcd0388e5d32 2024-11-21T01:46:19,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/a7b7b4daa5684c078721a2c673af545a is 50, key is test_row_0/B:col10/1732153578761/Put/seqid=0 2024-11-21T01:46:19,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742426_1602 (size=12151) 2024-11-21T01:46:19,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-21T01:46:19,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:19,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:19,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:19,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:19,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:19,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:19,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-21T01:46:19,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:19,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153639512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:19,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:19,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:19,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:19,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:19,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153639523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:19,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:19,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:19,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:19,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:19,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153639523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:19,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:19,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:19,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153639523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:19,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:19,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:19,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:19,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:19,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:19,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:19,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153639533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:19,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:19,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:19,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:19,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-21T01:46:19,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:19,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:19,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:19,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:19,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:19,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:19,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-21T01:46:19,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:19,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:19,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:19,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:19,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:19,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:19,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-21T01:46:19,762 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/a7b7b4daa5684c078721a2c673af545a 2024-11-21T01:46:19,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/f5d772f4f76442eca52b01a699cb3477 is 50, key is test_row_0/C:col10/1732153578761/Put/seqid=0 2024-11-21T01:46:19,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742427_1603 (size=12151) 2024-11-21T01:46:19,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/f5d772f4f76442eca52b01a699cb3477 2024-11-21T01:46:19,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/29b46d3d15794e3a9035fcd0388e5d32 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/29b46d3d15794e3a9035fcd0388e5d32 2024-11-21T01:46:19,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/29b46d3d15794e3a9035fcd0388e5d32, entries=150, sequenceid=141, filesize=30.4 K 2024-11-21T01:46:19,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/a7b7b4daa5684c078721a2c673af545a as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/a7b7b4daa5684c078721a2c673af545a 2024-11-21T01:46:19,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/a7b7b4daa5684c078721a2c673af545a, entries=150, sequenceid=141, filesize=11.9 K 2024-11-21T01:46:19,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/f5d772f4f76442eca52b01a699cb3477 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/f5d772f4f76442eca52b01a699cb3477 2024-11-21T01:46:19,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/f5d772f4f76442eca52b01a699cb3477, entries=150, sequenceid=141, filesize=11.9 K 2024-11-21T01:46:19,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~125.23 KB/128240, heapSize ~355.25 KB/363776, currentSize=53.67 KB/54960 for f1185a7567ce5119c4e675690fbe9d2b in 926ms, sequenceid=141, compaction requested=true 2024-11-21T01:46:19,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:19,805 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:46:19,806 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:46:19,806 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): f1185a7567ce5119c4e675690fbe9d2b/A is initiating minor compaction (all files) 2024-11-21T01:46:19,806 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f1185a7567ce5119c4e675690fbe9d2b/A in TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:19,806 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/c7fb1619a98643138da84776082d5de3, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/b7680dcff0e445c9b96f1288b552c993, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/29b46d3d15794e3a9035fcd0388e5d32] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp, totalSize=91.0 K 2024-11-21T01:46:19,806 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:19,806 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. files: [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/c7fb1619a98643138da84776082d5de3, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/b7680dcff0e445c9b96f1288b552c993, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/29b46d3d15794e3a9035fcd0388e5d32] 2024-11-21T01:46:19,806 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7fb1619a98643138da84776082d5de3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732153577525 2024-11-21T01:46:19,807 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7680dcff0e445c9b96f1288b552c993, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732153577539 2024-11-21T01:46:19,807 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29b46d3d15794e3a9035fcd0388e5d32, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732153578759 2024-11-21T01:46:19,812 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:19,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f1185a7567ce5119c4e675690fbe9d2b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:46:19,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:19,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f1185a7567ce5119c4e675690fbe9d2b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:46:19,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:19,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f1185a7567ce5119c4e675690fbe9d2b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:46:19,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:46:19,813 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:46:19,814 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:46:19,814 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): f1185a7567ce5119c4e675690fbe9d2b/B is initiating minor compaction (all files) 2024-11-21T01:46:19,815 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f1185a7567ce5119c4e675690fbe9d2b/B in TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:19,815 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/234a6f0c1fb843388278d844dd42726d, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/317de57286894d3dac7dd2eca9233857, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/a7b7b4daa5684c078721a2c673af545a] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp, totalSize=35.5 K 2024-11-21T01:46:19,815 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 234a6f0c1fb843388278d844dd42726d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732153577525 2024-11-21T01:46:19,815 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 317de57286894d3dac7dd2eca9233857, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732153577539 2024-11-21T01:46:19,816 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting a7b7b4daa5684c078721a2c673af545a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732153578759 2024-11-21T01:46:19,823 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411213e1add6f15e54d5b9ffb6b62f5e614ef_f1185a7567ce5119c4e675690fbe9d2b store=[table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:19,825 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411213e1add6f15e54d5b9ffb6b62f5e614ef_f1185a7567ce5119c4e675690fbe9d2b, store=[table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:19,825 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213e1add6f15e54d5b9ffb6b62f5e614ef_f1185a7567ce5119c4e675690fbe9d2b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:19,827 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f1185a7567ce5119c4e675690fbe9d2b#B#compaction#517 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:19,828 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/5eea9426c370417cb843860ad4b13726 is 50, key is test_row_0/B:col10/1732153578761/Put/seqid=0 2024-11-21T01:46:19,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742428_1604 (size=4469) 2024-11-21T01:46:19,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742429_1605 (size=12459) 2024-11-21T01:46:19,833 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f1185a7567ce5119c4e675690fbe9d2b#A#compaction#516 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:19,834 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/c3fa883a5386493bb1b6571e2fb4db47 is 175, key is test_row_0/A:col10/1732153578761/Put/seqid=0 2024-11-21T01:46:19,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742430_1606 (size=31413) 2024-11-21T01:46:19,838 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/5eea9426c370417cb843860ad4b13726 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/5eea9426c370417cb843860ad4b13726 2024-11-21T01:46:19,843 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f1185a7567ce5119c4e675690fbe9d2b/B of f1185a7567ce5119c4e675690fbe9d2b into 5eea9426c370417cb843860ad4b13726(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:19,843 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:19,843 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., storeName=f1185a7567ce5119c4e675690fbe9d2b/B, priority=13, startTime=1732153579813; duration=0sec 2024-11-21T01:46:19,843 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:46:19,843 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f1185a7567ce5119c4e675690fbe9d2b:B 2024-11-21T01:46:19,843 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:46:19,844 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/c3fa883a5386493bb1b6571e2fb4db47 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/c3fa883a5386493bb1b6571e2fb4db47 2024-11-21T01:46:19,844 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:46:19,844 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): f1185a7567ce5119c4e675690fbe9d2b/C is initiating minor compaction (all files) 2024-11-21T01:46:19,844 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f1185a7567ce5119c4e675690fbe9d2b/C in TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:19,845 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/34e19aec69894be687734e9e46e0d6e0, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/374fa21782d24f4bb3f69e59cb5e1366, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/f5d772f4f76442eca52b01a699cb3477] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp, totalSize=35.5 K 2024-11-21T01:46:19,845 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 34e19aec69894be687734e9e46e0d6e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732153577525 2024-11-21T01:46:19,845 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 374fa21782d24f4bb3f69e59cb5e1366, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732153577539 2024-11-21T01:46:19,846 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting f5d772f4f76442eca52b01a699cb3477, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732153578759 2024-11-21T01:46:19,850 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f1185a7567ce5119c4e675690fbe9d2b/A of f1185a7567ce5119c4e675690fbe9d2b into c3fa883a5386493bb1b6571e2fb4db47(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:19,850 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:19,850 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., storeName=f1185a7567ce5119c4e675690fbe9d2b/A, priority=13, startTime=1732153579805; duration=0sec 2024-11-21T01:46:19,850 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:19,850 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f1185a7567ce5119c4e675690fbe9d2b:A 2024-11-21T01:46:19,854 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f1185a7567ce5119c4e675690fbe9d2b#C#compaction#518 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:19,854 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/3f6313003d514f5f87283d694c787f75 is 50, key is test_row_0/C:col10/1732153578761/Put/seqid=0 2024-11-21T01:46:19,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742431_1607 (size=12459) 2024-11-21T01:46:19,890 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/3f6313003d514f5f87283d694c787f75 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/3f6313003d514f5f87283d694c787f75 2024-11-21T01:46:19,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-21T01:46:19,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:19,894 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing f1185a7567ce5119c4e675690fbe9d2b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-21T01:46:19,895 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f1185a7567ce5119c4e675690fbe9d2b/C of f1185a7567ce5119c4e675690fbe9d2b into 3f6313003d514f5f87283d694c787f75(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:19,895 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:19,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=A 2024-11-21T01:46:19,895 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., storeName=f1185a7567ce5119c4e675690fbe9d2b/C, priority=13, startTime=1732153579813; duration=0sec 2024-11-21T01:46:19,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:19,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=B 2024-11-21T01:46:19,895 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:19,895 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f1185a7567ce5119c4e675690fbe9d2b:C 2024-11-21T01:46:19,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:19,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=C 2024-11-21T01:46:19,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:19,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a87f1b049850421a8d5c0ee7f0c93a80_f1185a7567ce5119c4e675690fbe9d2b is 50, key is test_row_0/A:col10/1732153578888/Put/seqid=0 2024-11-21T01:46:19,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742432_1608 (size=12304) 2024-11-21T01:46:20,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:20,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:20,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:20,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:20,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153640060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:20,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153640061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:20,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:20,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153640059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:20,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:20,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153640062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:20,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:20,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153640067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:20,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:20,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:20,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:20,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732153580048, max timestamp=1732153580065Num uniques 859; 2024-11-21T01:46:20,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732153580048, max timestamp=1732153580065Num uniques 859; 2024-11-21T01:46:20,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732153580048, max timestamp=1732153580065Num uniques 859; 2024-11-21T01:46:20,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-21T01:46:20,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-21T01:46:20,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-21T01:46:20,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:20,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153640201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:20,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:20,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:20,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153640204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:20,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:20,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:20,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153640204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:20,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:20,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:20,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153640206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:20,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:20,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153640206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:20,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:20,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:20,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-21T01:46:20,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:20,311 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a87f1b049850421a8d5c0ee7f0c93a80_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a87f1b049850421a8d5c0ee7f0c93a80_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:20,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/4d2036a0977e4d4b830f7ea04cb720f1, store: [table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:20,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/4d2036a0977e4d4b830f7ea04cb720f1 is 175, key is test_row_0/A:col10/1732153578888/Put/seqid=0 2024-11-21T01:46:20,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742433_1609 (size=31105) 2024-11-21T01:46:20,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:20,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153640313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:20,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:20,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:20,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153640323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:20,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:20,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153640323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:20,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:20,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153640323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:20,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:20,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:20,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153640324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:20,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:20,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:20,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153640523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:20,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:20,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153640532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:20,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:20,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153640532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:20,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:20,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:20,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:20,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153640533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:20,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:20,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:20,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153640533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:20,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:20,720 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/4d2036a0977e4d4b830f7ea04cb720f1 2024-11-21T01:46:20,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/dea5bce0edbc48129338f3fe0f672f30 is 50, key is test_row_0/B:col10/1732153578888/Put/seqid=0 2024-11-21T01:46:20,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742434_1610 (size=12151) 2024-11-21T01:46:20,756 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/dea5bce0edbc48129338f3fe0f672f30 2024-11-21T01:46:20,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/c01ec94b780a4022b1c066f5a708f79e is 50, key is test_row_0/C:col10/1732153578888/Put/seqid=0 2024-11-21T01:46:20,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742435_1611 (size=12151) 2024-11-21T01:46:20,775 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/c01ec94b780a4022b1c066f5a708f79e 2024-11-21T01:46:20,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/4d2036a0977e4d4b830f7ea04cb720f1 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/4d2036a0977e4d4b830f7ea04cb720f1 2024-11-21T01:46:20,788 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/4d2036a0977e4d4b830f7ea04cb720f1, entries=150, sequenceid=155, filesize=30.4 K 2024-11-21T01:46:20,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/dea5bce0edbc48129338f3fe0f672f30 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/dea5bce0edbc48129338f3fe0f672f30 2024-11-21T01:46:20,800 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/dea5bce0edbc48129338f3fe0f672f30, entries=150, sequenceid=155, filesize=11.9 K 2024-11-21T01:46:20,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/c01ec94b780a4022b1c066f5a708f79e as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/c01ec94b780a4022b1c066f5a708f79e 2024-11-21T01:46:20,804 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/c01ec94b780a4022b1c066f5a708f79e, entries=150, sequenceid=155, filesize=11.9 K 2024-11-21T01:46:20,805 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=132.99 KB/136185 for f1185a7567ce5119c4e675690fbe9d2b in 912ms, sequenceid=155, compaction requested=false 2024-11-21T01:46:20,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:20,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:20,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-21T01:46:20,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-21T01:46:20,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-21T01:46:20,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6900 sec 2024-11-21T01:46:20,810 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 1.6960 sec 2024-11-21T01:46:20,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:20,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732153580198, max timestamp=1732153580835Num uniques 1009; 2024-11-21T01:46:20,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-21T01:46:20,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732153580198, max timestamp=1732153580835Num uniques 1009; 2024-11-21T01:46:20,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-21T01:46:20,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f1185a7567ce5119c4e675690fbe9d2b 3/3 column families, dataSize=64.46 KB heapSize=177.88 KB 2024-11-21T01:46:20,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=A 2024-11-21T01:46:20,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:46:20,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=B 2024-11-21T01:46:20,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:46:20,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=C 2024-11-21T01:46:20,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T01:46:20,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a1d79a3e50bd47a38f6876aa77c77c46_f1185a7567ce5119c4e675690fbe9d2b is 50, key is test_row_0/A:col10/1732153580835/Put/seqid=0 2024-11-21T01:46:20,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742436_1612 (size=12304) 2024-11-21T01:46:20,854 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:20,860 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a1d79a3e50bd47a38f6876aa77c77c46_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a1d79a3e50bd47a38f6876aa77c77c46_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:20,861 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/43ea0c3865914ae4b092ec1d6b17952c, store: [table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:20,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/43ea0c3865914ae4b092ec1d6b17952c is 175, key is test_row_0/A:col10/1732153580835/Put/seqid=0 2024-11-21T01:46:20,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742437_1613 (size=31105) 2024-11-21T01:46:20,868 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/43ea0c3865914ae4b092ec1d6b17952c 2024-11-21T01:46:20,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:20,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153640865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:20,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:20,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153640866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:20,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/e076d21e822a43d9853dfa57a91d757c is 50, key is test_row_0/B:col10/1732153580835/Put/seqid=0 2024-11-21T01:46:20,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:20,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153640868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:20,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:20,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:20,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153640870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:20,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:20,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742438_1614 (size=12151) 2024-11-21T01:46:20,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:20,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153640873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:20,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:20,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:20,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/e076d21e822a43d9853dfa57a91d757c 2024-11-21T01:46:20,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/0eeb9c48a73b44659f15451fb8d8269c is 50, key is test_row_0/C:col10/1732153580835/Put/seqid=0 2024-11-21T01:46:20,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742439_1615 (size=12151) 2024-11-21T01:46:20,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/0eeb9c48a73b44659f15451fb8d8269c 2024-11-21T01:46:20,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/43ea0c3865914ae4b092ec1d6b17952c as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/43ea0c3865914ae4b092ec1d6b17952c 2024-11-21T01:46:20,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/43ea0c3865914ae4b092ec1d6b17952c, entries=150, sequenceid=193, filesize=30.4 K 2024-11-21T01:46:20,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/e076d21e822a43d9853dfa57a91d757c as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/e076d21e822a43d9853dfa57a91d757c 2024-11-21T01:46:20,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/e076d21e822a43d9853dfa57a91d757c, entries=150, sequenceid=193, filesize=11.9 K 2024-11-21T01:46:20,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/0eeb9c48a73b44659f15451fb8d8269c as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/0eeb9c48a73b44659f15451fb8d8269c 2024-11-21T01:46:20,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/0eeb9c48a73b44659f15451fb8d8269c, entries=150, sequenceid=193, filesize=11.9 K 2024-11-21T01:46:20,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.46 KB/66005, heapSize ~177.84 KB/182104, currentSize=114.05 KB/116790 for f1185a7567ce5119c4e675690fbe9d2b in 114ms, sequenceid=193, compaction requested=true 2024-11-21T01:46:20,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:20,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f1185a7567ce5119c4e675690fbe9d2b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:46:20,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:20,953 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:46:20,953 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:46:20,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f1185a7567ce5119c4e675690fbe9d2b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:46:20,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:20,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f1185a7567ce5119c4e675690fbe9d2b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:46:20,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:46:20,954 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:46:20,954 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93623 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:46:20,954 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): f1185a7567ce5119c4e675690fbe9d2b/B is initiating minor compaction (all files) 2024-11-21T01:46:20,954 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): f1185a7567ce5119c4e675690fbe9d2b/A is initiating minor compaction (all files) 2024-11-21T01:46:20,954 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f1185a7567ce5119c4e675690fbe9d2b/B in TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:20,954 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f1185a7567ce5119c4e675690fbe9d2b/A in TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:20,954 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/5eea9426c370417cb843860ad4b13726, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/dea5bce0edbc48129338f3fe0f672f30, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/e076d21e822a43d9853dfa57a91d757c] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp, totalSize=35.9 K 2024-11-21T01:46:20,955 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/c3fa883a5386493bb1b6571e2fb4db47, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/4d2036a0977e4d4b830f7ea04cb720f1, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/43ea0c3865914ae4b092ec1d6b17952c] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp, totalSize=91.4 K 2024-11-21T01:46:20,955 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:20,955 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. files: [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/c3fa883a5386493bb1b6571e2fb4db47, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/4d2036a0977e4d4b830f7ea04cb720f1, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/43ea0c3865914ae4b092ec1d6b17952c] 2024-11-21T01:46:20,955 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 5eea9426c370417cb843860ad4b13726, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732153578759 2024-11-21T01:46:20,955 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting dea5bce0edbc48129338f3fe0f672f30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732153578885 2024-11-21T01:46:20,955 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3fa883a5386493bb1b6571e2fb4db47, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732153578759 2024-11-21T01:46:20,956 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting e076d21e822a43d9853dfa57a91d757c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732153580198 2024-11-21T01:46:20,956 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d2036a0977e4d4b830f7ea04cb720f1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732153578885 2024-11-21T01:46:20,956 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43ea0c3865914ae4b092ec1d6b17952c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732153580198 2024-11-21T01:46:20,964 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:20,964 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f1185a7567ce5119c4e675690fbe9d2b#B#compaction#525 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:20,965 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/8338fce7cddd4eb7804109ef5dc4fff5 is 50, key is test_row_0/B:col10/1732153580835/Put/seqid=0 2024-11-21T01:46:20,966 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121d9182a797309454687cf4c5cf55e3e67_f1185a7567ce5119c4e675690fbe9d2b store=[table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:20,967 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121d9182a797309454687cf4c5cf55e3e67_f1185a7567ce5119c4e675690fbe9d2b, store=[table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:20,968 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d9182a797309454687cf4c5cf55e3e67_f1185a7567ce5119c4e675690fbe9d2b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:20,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742441_1617 (size=4469) 2024-11-21T01:46:20,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:20,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f1185a7567ce5119c4e675690fbe9d2b 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-21T01:46:20,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=A 2024-11-21T01:46:20,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:46:20,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=B 2024-11-21T01:46:20,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:46:20,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=C 2024-11-21T01:46:20,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:46:20,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742440_1616 (size=12561) 2024-11-21T01:46:21,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:21,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153640996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:21,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:21,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153640997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:21,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:21,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:21,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121dfdf475b2ffe4aa7a9b9fa6e95a61dd5_f1185a7567ce5119c4e675690fbe9d2b is 50, key is test_row_0/A:col10/1732153580985/Put/seqid=0 2024-11-21T01:46:21,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:21,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153641002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:21,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:21,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:21,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153641003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:21,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:21,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153641004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:21,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:21,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:21,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742442_1618 (size=17284) 2024-11-21T01:46:21,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:21,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153641114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:21,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:21,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:21,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153641114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:21,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:21,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153641114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:21,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:21,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:21,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:21,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153641123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:21,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:21,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:21,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153641123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:21,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:21,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-21T01:46:21,266 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:46:21,266 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:46:21,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-21T01:46:21,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-21T01:46:21,269 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:46:21,270 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:46:21,270 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:46:21,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:21,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153641323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:21,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153641323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:21,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:21,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153641333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:21,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153641333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:21,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153641333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:21,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-21T01:46:21,381 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f1185a7567ce5119c4e675690fbe9d2b#A#compaction#526 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:21,382 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/838a58087ef840099a1a696ec642b10d is 175, key is test_row_0/A:col10/1732153580835/Put/seqid=0 2024-11-21T01:46:21,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742443_1619 (size=31515) 2024-11-21T01:46:21,400 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/8338fce7cddd4eb7804109ef5dc4fff5 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/8338fce7cddd4eb7804109ef5dc4fff5 2024-11-21T01:46:21,406 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f1185a7567ce5119c4e675690fbe9d2b/B of f1185a7567ce5119c4e675690fbe9d2b into 8338fce7cddd4eb7804109ef5dc4fff5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:21,406 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:21,406 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., storeName=f1185a7567ce5119c4e675690fbe9d2b/B, priority=13, startTime=1732153580953; duration=0sec 2024-11-21T01:46:21,407 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:46:21,407 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f1185a7567ce5119c4e675690fbe9d2b:B 2024-11-21T01:46:21,407 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:46:21,409 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:46:21,409 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): f1185a7567ce5119c4e675690fbe9d2b/C is initiating minor compaction (all files) 2024-11-21T01:46:21,409 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f1185a7567ce5119c4e675690fbe9d2b/C in TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:21,409 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/3f6313003d514f5f87283d694c787f75, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/c01ec94b780a4022b1c066f5a708f79e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/0eeb9c48a73b44659f15451fb8d8269c] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp, totalSize=35.9 K 2024-11-21T01:46:21,409 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f6313003d514f5f87283d694c787f75, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732153578759 2024-11-21T01:46:21,410 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting c01ec94b780a4022b1c066f5a708f79e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732153578885 2024-11-21T01:46:21,410 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 0eeb9c48a73b44659f15451fb8d8269c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732153580198 2024-11-21T01:46:21,418 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f1185a7567ce5119c4e675690fbe9d2b#C#compaction#528 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:21,419 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/ee2733f9297d488cbbee3201fdf51447 is 50, key is test_row_0/C:col10/1732153580835/Put/seqid=0 2024-11-21T01:46:21,422 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:21,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-21T01:46:21,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:21,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:21,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:21,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:21,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:21,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:21,428 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121dfdf475b2ffe4aa7a9b9fa6e95a61dd5_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121dfdf475b2ffe4aa7a9b9fa6e95a61dd5_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:21,430 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/b34ac67e0c304a7a8a3f556e3e05aba9, store: [table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:21,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/b34ac67e0c304a7a8a3f556e3e05aba9 is 175, key is test_row_0/A:col10/1732153580985/Put/seqid=0 2024-11-21T01:46:21,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742444_1620 (size=12561) 2024-11-21T01:46:21,453 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/ee2733f9297d488cbbee3201fdf51447 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/ee2733f9297d488cbbee3201fdf51447 2024-11-21T01:46:21,459 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f1185a7567ce5119c4e675690fbe9d2b/C of f1185a7567ce5119c4e675690fbe9d2b into ee2733f9297d488cbbee3201fdf51447(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:21,459 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:21,459 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., storeName=f1185a7567ce5119c4e675690fbe9d2b/C, priority=13, startTime=1732153580953; duration=0sec 2024-11-21T01:46:21,459 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:21,459 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f1185a7567ce5119c4e675690fbe9d2b:C 2024-11-21T01:46:21,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742445_1621 (size=48389) 2024-11-21T01:46:21,462 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=215, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/b34ac67e0c304a7a8a3f556e3e05aba9 2024-11-21T01:46:21,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/56ac2f2f78c844d8ad74b87f9d923068 is 50, key is test_row_0/B:col10/1732153580985/Put/seqid=0 2024-11-21T01:46:21,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742446_1622 (size=12151) 2024-11-21T01:46:21,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-21T01:46:21,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:21,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:21,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:21,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:21,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:21,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:21,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-21T01:46:21,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153641632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:21,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:21,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153641633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:21,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:21,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:21,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153641649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:21,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153641649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:21,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:21,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153641650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:21,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:21,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:21,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:21,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-21T01:46:21,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:21,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:21,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:21,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:21,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:21,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:21,795 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/838a58087ef840099a1a696ec642b10d as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/838a58087ef840099a1a696ec642b10d 2024-11-21T01:46:21,800 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f1185a7567ce5119c4e675690fbe9d2b/A of f1185a7567ce5119c4e675690fbe9d2b into 838a58087ef840099a1a696ec642b10d(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:21,800 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:21,800 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., storeName=f1185a7567ce5119c4e675690fbe9d2b/A, priority=13, startTime=1732153580953; duration=0sec 2024-11-21T01:46:21,800 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:21,800 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f1185a7567ce5119c4e675690fbe9d2b:A 2024-11-21T01:46:21,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/56ac2f2f78c844d8ad74b87f9d923068 2024-11-21T01:46:21,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/f8ada7251a224ad3b45add5b6b9ac307 is 50, key is test_row_0/C:col10/1732153580985/Put/seqid=0 2024-11-21T01:46:21,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742447_1623 (size=12151) 2024-11-21T01:46:21,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-21T01:46:21,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:21,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:21,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:21,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:21,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:21,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-21T01:46:22,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-21T01:46:22,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:22,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:22,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:22,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:22,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:22,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:22,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153642143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:22,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:22,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:22,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:22,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153642152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:22,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:22,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:22,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:22,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153642163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:22,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:22,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153642163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:22,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:22,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:22,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:22,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:22,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:22,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153642163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:22,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:22,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:22,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:22,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-21T01:46:22,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:22,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:22,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:22,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:22,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:22,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/f8ada7251a224ad3b45add5b6b9ac307 2024-11-21T01:46:22,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/b34ac67e0c304a7a8a3f556e3e05aba9 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/b34ac67e0c304a7a8a3f556e3e05aba9 2024-11-21T01:46:22,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/b34ac67e0c304a7a8a3f556e3e05aba9, entries=250, sequenceid=215, filesize=47.3 K 2024-11-21T01:46:22,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/56ac2f2f78c844d8ad74b87f9d923068 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/56ac2f2f78c844d8ad74b87f9d923068 2024-11-21T01:46:22,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/56ac2f2f78c844d8ad74b87f9d923068, entries=150, sequenceid=215, filesize=11.9 K 2024-11-21T01:46:22,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/f8ada7251a224ad3b45add5b6b9ac307 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/f8ada7251a224ad3b45add5b6b9ac307 2024-11-21T01:46:22,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/f8ada7251a224ad3b45add5b6b9ac307, entries=150, sequenceid=215, filesize=11.9 K 2024-11-21T01:46:22,335 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=53.67 KB/54960 for f1185a7567ce5119c4e675690fbe9d2b in 1348ms, sequenceid=215, compaction requested=false 2024-11-21T01:46:22,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:22,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-21T01:46:22,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:22,346 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing f1185a7567ce5119c4e675690fbe9d2b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-21T01:46:22,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=A 2024-11-21T01:46:22,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:22,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=B 2024-11-21T01:46:22,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:22,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=C 2024-11-21T01:46:22,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:22,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411217ecaacd968804c9fa1377ae3fa7bd7c4_f1185a7567ce5119c4e675690fbe9d2b is 50, key is test_row_0/A:col10/1732153580988/Put/seqid=0 2024-11-21T01:46:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-21T01:46:22,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742448_1624 (size=12304) 2024-11-21T01:46:22,632 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T01:46:22,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,831 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411217ecaacd968804c9fa1377ae3fa7bd7c4_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217ecaacd968804c9fa1377ae3fa7bd7c4_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:22,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/3bf9800d6fd5416ea669e1103fbc7884, store: [table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:22,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/3bf9800d6fd5416ea669e1103fbc7884 is 175, key is test_row_0/A:col10/1732153580988/Put/seqid=0 2024-11-21T01:46:22,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742449_1625 (size=31105) 2024-11-21T01:46:22,852 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=229, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/3bf9800d6fd5416ea669e1103fbc7884 2024-11-21T01:46:22,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/1641efd28c18412d908e4d51ef3d4f59 is 50, key is test_row_0/B:col10/1732153580988/Put/seqid=0 2024-11-21T01:46:22,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742450_1626 (size=12151) 2024-11-21T01:46:22,875 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/1641efd28c18412d908e4d51ef3d4f59 2024-11-21T01:46:22,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/f48591b9efe743db8bb3057b7c5f0141 is 50, key is test_row_0/C:col10/1732153580988/Put/seqid=0 2024-11-21T01:46:22,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742451_1627 (size=12151) 2024-11-21T01:46:22,919 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/f48591b9efe743db8bb3057b7c5f0141 2024-11-21T01:46:22,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/3bf9800d6fd5416ea669e1103fbc7884 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/3bf9800d6fd5416ea669e1103fbc7884 2024-11-21T01:46:22,931 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/3bf9800d6fd5416ea669e1103fbc7884, entries=150, sequenceid=229, filesize=30.4 K 2024-11-21T01:46:22,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/1641efd28c18412d908e4d51ef3d4f59 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/1641efd28c18412d908e4d51ef3d4f59 2024-11-21T01:46:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,936 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/1641efd28c18412d908e4d51ef3d4f59, entries=150, sequenceid=229, filesize=11.9 K 2024-11-21T01:46:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/f48591b9efe743db8bb3057b7c5f0141 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/f48591b9efe743db8bb3057b7c5f0141 2024-11-21T01:46:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,943 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/f48591b9efe743db8bb3057b7c5f0141, entries=150, sequenceid=229, filesize=11.9 K 2024-11-21T01:46:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,944 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for f1185a7567ce5119c4e675690fbe9d2b in 598ms, sequenceid=229, compaction requested=true 2024-11-21T01:46:22,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:22,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:22,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-21T01:46:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-21T01:46:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-21T01:46:22,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6760 sec 2024-11-21T01:46:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 1.6840 sec 2024-11-21T01:46:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:23,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f1185a7567ce5119c4e675690fbe9d2b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T01:46:23,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=A 2024-11-21T01:46:23,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:23,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=B 2024-11-21T01:46:23,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:23,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=C 2024-11-21T01:46:23,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121cc0cac1762414e14b9b53cb413e6a9ed_f1185a7567ce5119c4e675690fbe9d2b is 50, key is test_row_0/A:col10/1732153583207/Put/seqid=0 2024-11-21T01:46:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742452_1628 (size=17284) 2024-11-21T01:46:23,273 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:23,277 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121cc0cac1762414e14b9b53cb413e6a9ed_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cc0cac1762414e14b9b53cb413e6a9ed_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:23,279 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/804c6ee2e9714b3db36fe2bbe94694c8, store: [table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:23,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/804c6ee2e9714b3db36fe2bbe94694c8 is 175, key is test_row_0/A:col10/1732153583207/Put/seqid=0 2024-11-21T01:46:23,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732153583272, max timestamp=1732153583281Num uniques 859; 2024-11-21T01:46:23,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732153583272, max timestamp=1732153583281Num uniques 859; 2024-11-21T01:46:23,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T01:46:23,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T01:46:23,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742453_1629 (size=48385) 2024-11-21T01:46:23,289 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/804c6ee2e9714b3db36fe2bbe94694c8 2024-11-21T01:46:23,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/5e4ed402d40a427aaad3b429c4d6daee is 50, key is test_row_0/B:col10/1732153583207/Put/seqid=0 2024-11-21T01:46:23,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153643299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:23,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:23,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:23,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153643301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:23,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:23,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:23,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153643304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:23,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:23,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153643307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:23,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:23,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153643307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:23,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:23,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:23,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742454_1630 (size=9757) 2024-11-21T01:46:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-21T01:46:23,413 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:46:23,414 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:46:23,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees 2024-11-21T01:46:23,416 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:46:23,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:23,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-21T01:46:23,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153643413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:23,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:23,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153643413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:23,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:23,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:23,417 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:46:23,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:46:23,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153643423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:23,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:23,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153643424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:23,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:23,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153643424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:23,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:23,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:23,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-21T01:46:23,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-21T01:46:23,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:23,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:23,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:23,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:23,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:23,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:23,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:23,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153643623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:23,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:23,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:23,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153643623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:23,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:23,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:23,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153643633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:23,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:23,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153643633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:23,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:23,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:23,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:23,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153643633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:23,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:23,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-21T01:46:23,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:23,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:23,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:23,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:23,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:23,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-21T01:46:23,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/5e4ed402d40a427aaad3b429c4d6daee 2024-11-21T01:46:23,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/c9afeb9a2abd4216adfcbca78b01ebf0 is 50, key is test_row_0/C:col10/1732153583207/Put/seqid=0 2024-11-21T01:46:23,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742455_1631 (size=9757) 2024-11-21T01:46:23,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-21T01:46:23,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:23,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:23,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:23,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:23,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:23,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153643933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:23,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:23,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153643935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:23,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:23,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153643943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:23,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:23,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153643943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:23,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:23,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:23,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153643943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:23,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:23,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:24,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-21T01:46:24,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:24,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:24,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:24,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:24,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-21T01:46:24,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/c9afeb9a2abd4216adfcbca78b01ebf0 2024-11-21T01:46:24,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/804c6ee2e9714b3db36fe2bbe94694c8 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/804c6ee2e9714b3db36fe2bbe94694c8 2024-11-21T01:46:24,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/804c6ee2e9714b3db36fe2bbe94694c8, entries=250, sequenceid=239, filesize=47.3 K 2024-11-21T01:46:24,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/5e4ed402d40a427aaad3b429c4d6daee as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/5e4ed402d40a427aaad3b429c4d6daee 2024-11-21T01:46:24,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/5e4ed402d40a427aaad3b429c4d6daee, entries=100, sequenceid=239, filesize=9.5 K 2024-11-21T01:46:24,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/c9afeb9a2abd4216adfcbca78b01ebf0 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/c9afeb9a2abd4216adfcbca78b01ebf0 2024-11-21T01:46:24,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-21T01:46:24,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:24,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:24,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:24,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:24,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:24,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/c9afeb9a2abd4216adfcbca78b01ebf0, entries=100, sequenceid=239, filesize=9.5 K 2024-11-21T01:46:24,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:24,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.94 KB/135110 for f1185a7567ce5119c4e675690fbe9d2b in 960ms, sequenceid=239, compaction requested=true 2024-11-21T01:46:24,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:24,188 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:46:24,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f1185a7567ce5119c4e675690fbe9d2b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:46:24,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:24,188 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:46:24,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f1185a7567ce5119c4e675690fbe9d2b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:46:24,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:24,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f1185a7567ce5119c4e675690fbe9d2b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:46:24,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:46:24,189 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 159394 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:46:24,189 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): f1185a7567ce5119c4e675690fbe9d2b/A is initiating minor compaction (all files) 2024-11-21T01:46:24,189 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f1185a7567ce5119c4e675690fbe9d2b/A in TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:24,189 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/838a58087ef840099a1a696ec642b10d, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/b34ac67e0c304a7a8a3f556e3e05aba9, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/3bf9800d6fd5416ea669e1103fbc7884, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/804c6ee2e9714b3db36fe2bbe94694c8] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp, totalSize=155.7 K 2024-11-21T01:46:24,189 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:24,189 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. files: [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/838a58087ef840099a1a696ec642b10d, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/b34ac67e0c304a7a8a3f556e3e05aba9, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/3bf9800d6fd5416ea669e1103fbc7884, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/804c6ee2e9714b3db36fe2bbe94694c8] 2024-11-21T01:46:24,190 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 838a58087ef840099a1a696ec642b10d, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732153580198 2024-11-21T01:46:24,194 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46620 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:46:24,194 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): f1185a7567ce5119c4e675690fbe9d2b/B is initiating minor compaction (all files) 2024-11-21T01:46:24,194 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f1185a7567ce5119c4e675690fbe9d2b/B in TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:24,194 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/8338fce7cddd4eb7804109ef5dc4fff5, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/56ac2f2f78c844d8ad74b87f9d923068, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/1641efd28c18412d908e4d51ef3d4f59, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/5e4ed402d40a427aaad3b429c4d6daee] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp, totalSize=45.5 K 2024-11-21T01:46:24,195 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting b34ac67e0c304a7a8a3f556e3e05aba9, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732153580866 2024-11-21T01:46:24,195 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 8338fce7cddd4eb7804109ef5dc4fff5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732153580198 2024-11-21T01:46:24,197 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bf9800d6fd5416ea669e1103fbc7884, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732153580988 2024-11-21T01:46:24,198 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 56ac2f2f78c844d8ad74b87f9d923068, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732153580866 2024-11-21T01:46:24,206 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 804c6ee2e9714b3db36fe2bbe94694c8, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732153583195 2024-11-21T01:46:24,206 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 1641efd28c18412d908e4d51ef3d4f59, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732153580988 2024-11-21T01:46:24,206 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e4ed402d40a427aaad3b429c4d6daee, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732153583207 2024-11-21T01:46:24,220 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f1185a7567ce5119c4e675690fbe9d2b#B#compaction#537 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:24,220 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/f68223045ee54355a0366b7ab35174a0 is 50, key is test_row_0/B:col10/1732153583207/Put/seqid=0 2024-11-21T01:46:24,258 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:24,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742456_1632 (size=12697) 2024-11-21T01:46:24,285 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121c234b339590840c19e650390b931c6a5_f1185a7567ce5119c4e675690fbe9d2b store=[table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:24,288 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121c234b339590840c19e650390b931c6a5_f1185a7567ce5119c4e675690fbe9d2b, store=[table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:24,288 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c234b339590840c19e650390b931c6a5_f1185a7567ce5119c4e675690fbe9d2b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:24,295 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/f68223045ee54355a0366b7ab35174a0 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/f68223045ee54355a0366b7ab35174a0 2024-11-21T01:46:24,307 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f1185a7567ce5119c4e675690fbe9d2b/B of f1185a7567ce5119c4e675690fbe9d2b into f68223045ee54355a0366b7ab35174a0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:24,307 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:24,307 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., storeName=f1185a7567ce5119c4e675690fbe9d2b/B, priority=12, startTime=1732153584188; duration=0sec 2024-11-21T01:46:24,307 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:46:24,307 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f1185a7567ce5119c4e675690fbe9d2b:B 2024-11-21T01:46:24,307 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T01:46:24,308 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46620 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T01:46:24,308 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): f1185a7567ce5119c4e675690fbe9d2b/C is initiating minor compaction (all files) 2024-11-21T01:46:24,309 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f1185a7567ce5119c4e675690fbe9d2b/C in TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:24,309 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/ee2733f9297d488cbbee3201fdf51447, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/f8ada7251a224ad3b45add5b6b9ac307, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/f48591b9efe743db8bb3057b7c5f0141, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/c9afeb9a2abd4216adfcbca78b01ebf0] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp, totalSize=45.5 K 2024-11-21T01:46:24,309 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting ee2733f9297d488cbbee3201fdf51447, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732153580198 2024-11-21T01:46:24,309 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting f8ada7251a224ad3b45add5b6b9ac307, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732153580866 2024-11-21T01:46:24,310 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting f48591b9efe743db8bb3057b7c5f0141, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732153580988 2024-11-21T01:46:24,310 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting c9afeb9a2abd4216adfcbca78b01ebf0, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732153583207 2024-11-21T01:46:24,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742457_1633 (size=4469) 2024-11-21T01:46:24,336 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f1185a7567ce5119c4e675690fbe9d2b#C#compaction#539 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:24,336 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/c7a15f8d2a174a7897e680a35349a6b4 is 50, key is test_row_0/C:col10/1732153583207/Put/seqid=0 2024-11-21T01:46:24,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-21T01:46:24,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:24,350 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2902): Flushing f1185a7567ce5119c4e675690fbe9d2b 3/3 column families, dataSize=131.94 KB heapSize=383.31 KB 2024-11-21T01:46:24,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=A 2024-11-21T01:46:24,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:46:24,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=B 2024-11-21T01:46:24,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:46:24,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=C 2024-11-21T01:46:24,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:46:24,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742458_1634 (size=12697) 2024-11-21T01:46:24,370 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/c7a15f8d2a174a7897e680a35349a6b4 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/c7a15f8d2a174a7897e680a35349a6b4 2024-11-21T01:46:24,378 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f1185a7567ce5119c4e675690fbe9d2b/C of f1185a7567ce5119c4e675690fbe9d2b into c7a15f8d2a174a7897e680a35349a6b4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:24,378 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:24,378 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., storeName=f1185a7567ce5119c4e675690fbe9d2b/C, priority=12, startTime=1732153584189; duration=0sec 2024-11-21T01:46:24,378 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:24,378 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f1185a7567ce5119c4e675690fbe9d2b:C 2024-11-21T01:46:24,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213ed6341309a04bb8a54ad206d095b2d1_f1185a7567ce5119c4e675690fbe9d2b is 50, key is test_row_0/A:col10/1732153583304/Put/seqid=0 2024-11-21T01:46:24,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742459_1635 (size=12454) 2024-11-21T01:46:24,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:24,424 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213ed6341309a04bb8a54ad206d095b2d1_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213ed6341309a04bb8a54ad206d095b2d1_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:24,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/09e0b535ff854c038c66d5fb0f8d87da, store: [table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:24,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/09e0b535ff854c038c66d5fb0f8d87da is 175, key is test_row_0/A:col10/1732153583304/Put/seqid=0 2024-11-21T01:46:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:24,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:24,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742460_1636 (size=31255) 2024-11-21T01:46:24,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153644475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:24,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:24,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153644482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:24,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153644485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:24,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153644485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:24,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:24,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:24,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:24,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:24,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153644482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:24,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:24,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:24,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:24,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:24,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:24,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:24,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:24,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:24,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-21T01:46:24,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153644596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:24,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:24,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:24,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:24,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153644603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:24,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:24,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153644604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:24,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:24,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:24,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:24,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153644604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:24,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:24,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:24,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153644605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:24,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:24,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:24,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:24,720 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f1185a7567ce5119c4e675690fbe9d2b#A#compaction#538 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:24,721 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/bea94f6e7f114d66b4a13dba475689f2 is 175, key is test_row_0/A:col10/1732153583207/Put/seqid=0 2024-11-21T01:46:24,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742461_1637 (size=31758) 2024-11-21T01:46:24,750 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/bea94f6e7f114d66b4a13dba475689f2 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/bea94f6e7f114d66b4a13dba475689f2 2024-11-21T01:46:24,759 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f1185a7567ce5119c4e675690fbe9d2b/A of f1185a7567ce5119c4e675690fbe9d2b into bea94f6e7f114d66b4a13dba475689f2(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:24,759 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:24,759 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., storeName=f1185a7567ce5119c4e675690fbe9d2b/A, priority=12, startTime=1732153584188; duration=0sec 2024-11-21T01:46:24,759 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:24,759 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f1185a7567ce5119c4e675690fbe9d2b:A 2024-11-21T01:46:24,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153644813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:24,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153644813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:24,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153644813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:24,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153644813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:24,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:24,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:24,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:24,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:24,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:24,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:24,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:24,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:24,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:24,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153644823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:24,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:24,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:24,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:24,854 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/09e0b535ff854c038c66d5fb0f8d87da 2024-11-21T01:46:24,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/ded45e31221c40419d1d252b5c6b0dc2 is 50, key is test_row_0/B:col10/1732153583304/Put/seqid=0 2024-11-21T01:46:24,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742462_1638 (size=12301) 2024-11-21T01:46:25,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:25,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153645123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:25,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:25,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:25,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:25,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153645123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:25,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:25,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:25,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:25,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:25,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:25,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153645133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:25,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:25,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153645133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:25,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:25,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153645133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:25,310 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/ded45e31221c40419d1d252b5c6b0dc2 2024-11-21T01:46:25,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/723a907395324090bcd71bb1e7ffb608 is 50, key is test_row_0/C:col10/1732153583304/Put/seqid=0 2024-11-21T01:46:25,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742463_1639 (size=12301) 2024-11-21T01:46:25,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-21T01:46:25,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:25,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153645633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:25,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:25,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:25,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:25,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153645633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:25,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:25,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:25,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:25,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:25,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:25,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153645643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:25,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:25,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153645643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:25,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:25,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:25,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:25,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:25,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:25,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:25,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:25,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153645657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:25,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:25,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:25,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:25,746 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/723a907395324090bcd71bb1e7ffb608 2024-11-21T01:46:25,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/09e0b535ff854c038c66d5fb0f8d87da as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/09e0b535ff854c038c66d5fb0f8d87da 2024-11-21T01:46:25,755 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/09e0b535ff854c038c66d5fb0f8d87da, entries=150, sequenceid=272, filesize=30.5 K 2024-11-21T01:46:25,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/ded45e31221c40419d1d252b5c6b0dc2 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/ded45e31221c40419d1d252b5c6b0dc2 2024-11-21T01:46:25,760 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/ded45e31221c40419d1d252b5c6b0dc2, entries=150, sequenceid=272, filesize=12.0 K 2024-11-21T01:46:25,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/723a907395324090bcd71bb1e7ffb608 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/723a907395324090bcd71bb1e7ffb608 2024-11-21T01:46:25,783 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/723a907395324090bcd71bb1e7ffb608, entries=150, sequenceid=272, filesize=12.0 K 2024-11-21T01:46:25,784 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~383.27 KB/392464, currentSize=53.67 KB/54960 for f1185a7567ce5119c4e675690fbe9d2b in 1434ms, sequenceid=272, compaction requested=false 2024-11-21T01:46:25,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2603): Flush status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:25,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:25,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=173 2024-11-21T01:46:25,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=173 2024-11-21T01:46:25,788 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-21T01:46:25,788 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3680 sec 2024-11-21T01:46:25,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees in 2.3750 sec 2024-11-21T01:46:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:26,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f1185a7567ce5119c4e675690fbe9d2b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T01:46:26,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=A 2024-11-21T01:46:26,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:26,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=B 2024-11-21T01:46:26,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:26,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=C 2024-11-21T01:46:26,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:26,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215810ae39f36c43c5acc8ef2b27985c19_f1185a7567ce5119c4e675690fbe9d2b is 50, key is test_row_0/A:col10/1732153584473/Put/seqid=0 2024-11-21T01:46:26,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742464_1640 (size=14994) 2024-11-21T01:46:26,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153646725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:26,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:26,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153646725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:26,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:26,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:26,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153646725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:26,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:26,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:26,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:26,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:26,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153646726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:26,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:26,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:26,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:26,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:26,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:26,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153646726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:26,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:26,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:26,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:26,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:26,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153646843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:26,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:26,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:26,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153646843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:26,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:26,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:26,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:26,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:26,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:26,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153646844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:26,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:26,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:26,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:26,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153646844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:26,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153646844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:27,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153647053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:27,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153647054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:27,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153647054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:27,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153647055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:27,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153647055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:27,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:27,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:27,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:27,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:27,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:27,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:27,075 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:27,080 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215810ae39f36c43c5acc8ef2b27985c19_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215810ae39f36c43c5acc8ef2b27985c19_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:27,081 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/41670f50f23748efaa2a358c8c1cf1c3, store: [table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:27,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/41670f50f23748efaa2a358c8c1cf1c3 is 175, key is test_row_0/A:col10/1732153584473/Put/seqid=0 2024-11-21T01:46:27,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742465_1641 (size=39949) 2024-11-21T01:46:27,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:27,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153647363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:27,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:27,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:27,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:27,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:27,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:27,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:27,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153647363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:27,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153647363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:27,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153647363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:27,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:27,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:27,369 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:27,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:27,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153647364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:27,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:27,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:27,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:27,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:27,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:27,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:27,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:27,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:27,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:27,489 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=286, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/41670f50f23748efaa2a358c8c1cf1c3 2024-11-21T01:46:27,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/3a87aed54d794522ad0ad9c264a497f3 is 50, key is test_row_0/B:col10/1732153584473/Put/seqid=0 2024-11-21T01:46:27,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742466_1642 (size=12301) 2024-11-21T01:46:27,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/3a87aed54d794522ad0ad9c264a497f3 2024-11-21T01:46:27,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/8a4f0c62e926461ebbc254b7b28ba9e6 is 50, key is test_row_0/C:col10/1732153584473/Put/seqid=0 2024-11-21T01:46:27,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742467_1643 (size=12301) 2024-11-21T01:46:27,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/8a4f0c62e926461ebbc254b7b28ba9e6 2024-11-21T01:46:27,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/41670f50f23748efaa2a358c8c1cf1c3 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/41670f50f23748efaa2a358c8c1cf1c3 2024-11-21T01:46:27,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/41670f50f23748efaa2a358c8c1cf1c3, entries=200, sequenceid=286, filesize=39.0 K 2024-11-21T01:46:27,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/3a87aed54d794522ad0ad9c264a497f3 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/3a87aed54d794522ad0ad9c264a497f3 2024-11-21T01:46:27,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/3a87aed54d794522ad0ad9c264a497f3, entries=150, sequenceid=286, filesize=12.0 K 2024-11-21T01:46:27,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/8a4f0c62e926461ebbc254b7b28ba9e6 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/8a4f0c62e926461ebbc254b7b28ba9e6 2024-11-21T01:46:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-21T01:46:27,593 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:46:27,593 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:46:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees 2024-11-21T01:46:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-21T01:46:27,596 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:46:27,596 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:46:27,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:46:27,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/8a4f0c62e926461ebbc254b7b28ba9e6, entries=150, sequenceid=286, filesize=12.0 K 2024-11-21T01:46:27,597 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f1185a7567ce5119c4e675690fbe9d2b in 953ms, sequenceid=286, compaction requested=true 2024-11-21T01:46:27,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:27,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f1185a7567ce5119c4e675690fbe9d2b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:46:27,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:27,598 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:46:27,598 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:46:27,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f1185a7567ce5119c4e675690fbe9d2b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:46:27,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:27,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f1185a7567ce5119c4e675690fbe9d2b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:46:27,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:46:27,602 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102962 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:46:27,602 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:46:27,602 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): f1185a7567ce5119c4e675690fbe9d2b/A is initiating minor compaction (all files) 2024-11-21T01:46:27,602 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): f1185a7567ce5119c4e675690fbe9d2b/B is initiating minor compaction (all files) 2024-11-21T01:46:27,602 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f1185a7567ce5119c4e675690fbe9d2b/A in TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:27,602 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f1185a7567ce5119c4e675690fbe9d2b/B in TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:27,602 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/f68223045ee54355a0366b7ab35174a0, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/ded45e31221c40419d1d252b5c6b0dc2, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/3a87aed54d794522ad0ad9c264a497f3] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp, totalSize=36.4 K 2024-11-21T01:46:27,602 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/bea94f6e7f114d66b4a13dba475689f2, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/09e0b535ff854c038c66d5fb0f8d87da, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/41670f50f23748efaa2a358c8c1cf1c3] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp, totalSize=100.5 K 2024-11-21T01:46:27,602 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:27,602 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. files: [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/bea94f6e7f114d66b4a13dba475689f2, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/09e0b535ff854c038c66d5fb0f8d87da, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/41670f50f23748efaa2a358c8c1cf1c3] 2024-11-21T01:46:27,602 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting bea94f6e7f114d66b4a13dba475689f2, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732153581000 2024-11-21T01:46:27,602 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting f68223045ee54355a0366b7ab35174a0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732153581000 2024-11-21T01:46:27,603 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting ded45e31221c40419d1d252b5c6b0dc2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732153583290 2024-11-21T01:46:27,603 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09e0b535ff854c038c66d5fb0f8d87da, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732153583290 2024-11-21T01:46:27,604 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a87aed54d794522ad0ad9c264a497f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732153584472 2024-11-21T01:46:27,604 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41670f50f23748efaa2a358c8c1cf1c3, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732153584472 2024-11-21T01:46:27,617 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f1185a7567ce5119c4e675690fbe9d2b#B#compaction#546 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:27,618 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/3702d53c68ce4dc0be534f1c0b376447 is 50, key is test_row_0/B:col10/1732153584473/Put/seqid=0 2024-11-21T01:46:27,622 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:27,646 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121b26af28ce2b1441dab8cd7353f06a8b6_f1185a7567ce5119c4e675690fbe9d2b store=[table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:27,648 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121b26af28ce2b1441dab8cd7353f06a8b6_f1185a7567ce5119c4e675690fbe9d2b, store=[table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:27,648 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b26af28ce2b1441dab8cd7353f06a8b6_f1185a7567ce5119c4e675690fbe9d2b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:27,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742468_1644 (size=12949) 2024-11-21T01:46:27,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742469_1645 (size=4469) 2024-11-21T01:46:27,664 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f1185a7567ce5119c4e675690fbe9d2b#A#compaction#547 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:27,665 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/562af2bd4984438d8fdba0d36d9b13d9 is 175, key is test_row_0/A:col10/1732153584473/Put/seqid=0 2024-11-21T01:46:27,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742470_1646 (size=31903) 2024-11-21T01:46:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-21T01:46:27,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-21T01:46:27,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:27,750 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2902): Flushing f1185a7567ce5119c4e675690fbe9d2b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T01:46:27,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=A 2024-11-21T01:46:27,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:27,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=B 2024-11-21T01:46:27,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:27,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=C 2024-11-21T01:46:27,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:27,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c053839637a74214ab56c2aacdcddea6_f1185a7567ce5119c4e675690fbe9d2b is 50, key is test_row_0/A:col10/1732153586724/Put/seqid=0 2024-11-21T01:46:27,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742471_1647 (size=12454) 2024-11-21T01:46:27,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:27,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:27,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153647890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:27,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153647892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:27,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:27,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:27,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153647893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:27,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:27,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:27,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:27,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:27,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:27,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:27,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:27,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:27,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:27,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153647894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:27,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153647896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:27,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:27,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:27,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:27,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:27,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:27,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-21T01:46:28,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:28,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153648013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:28,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:28,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:28,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:28,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:28,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153648013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:28,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:28,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153648013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:28,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:28,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:28,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:28,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153648013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:28,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:28,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:28,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:28,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:28,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:28,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:28,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153648013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:28,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:28,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:28,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:28,073 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/3702d53c68ce4dc0be534f1c0b376447 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/3702d53c68ce4dc0be534f1c0b376447 2024-11-21T01:46:28,081 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f1185a7567ce5119c4e675690fbe9d2b/B of f1185a7567ce5119c4e675690fbe9d2b into 3702d53c68ce4dc0be534f1c0b376447(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:28,081 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/562af2bd4984438d8fdba0d36d9b13d9 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/562af2bd4984438d8fdba0d36d9b13d9 2024-11-21T01:46:28,081 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:28,081 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., storeName=f1185a7567ce5119c4e675690fbe9d2b/B, priority=13, startTime=1732153587598; duration=0sec 2024-11-21T01:46:28,082 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:46:28,082 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f1185a7567ce5119c4e675690fbe9d2b:B 2024-11-21T01:46:28,082 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T01:46:28,083 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T01:46:28,083 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): f1185a7567ce5119c4e675690fbe9d2b/C is initiating minor compaction (all files) 2024-11-21T01:46:28,083 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f1185a7567ce5119c4e675690fbe9d2b/C in TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:28,083 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/c7a15f8d2a174a7897e680a35349a6b4, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/723a907395324090bcd71bb1e7ffb608, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/8a4f0c62e926461ebbc254b7b28ba9e6] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp, totalSize=36.4 K 2024-11-21T01:46:28,083 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting c7a15f8d2a174a7897e680a35349a6b4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732153581000 2024-11-21T01:46:28,083 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 723a907395324090bcd71bb1e7ffb608, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732153583290 2024-11-21T01:46:28,084 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a4f0c62e926461ebbc254b7b28ba9e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732153584472 2024-11-21T01:46:28,087 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f1185a7567ce5119c4e675690fbe9d2b/A of f1185a7567ce5119c4e675690fbe9d2b into 562af2bd4984438d8fdba0d36d9b13d9(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:28,087 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:28,088 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., storeName=f1185a7567ce5119c4e675690fbe9d2b/A, priority=13, startTime=1732153587598; duration=0sec 2024-11-21T01:46:28,088 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:28,088 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f1185a7567ce5119c4e675690fbe9d2b:A 2024-11-21T01:46:28,091 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f1185a7567ce5119c4e675690fbe9d2b#C#compaction#549 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:28,091 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/dfb37d1712b84f378505ca723a3c15a3 is 50, key is test_row_0/C:col10/1732153584473/Put/seqid=0 2024-11-21T01:46:28,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742472_1648 (size=12949) 2024-11-21T01:46:28,121 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/dfb37d1712b84f378505ca723a3c15a3 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/dfb37d1712b84f378505ca723a3c15a3 2024-11-21T01:46:28,126 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f1185a7567ce5119c4e675690fbe9d2b/C of f1185a7567ce5119c4e675690fbe9d2b into dfb37d1712b84f378505ca723a3c15a3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:28,126 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:28,126 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., storeName=f1185a7567ce5119c4e675690fbe9d2b/C, priority=13, startTime=1732153587598; duration=0sec 2024-11-21T01:46:28,126 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:28,126 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f1185a7567ce5119c4e675690fbe9d2b:C 2024-11-21T01:46:28,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:28,177 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c053839637a74214ab56c2aacdcddea6_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c053839637a74214ab56c2aacdcddea6_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:28,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/2308a93adff248cfad55b97c8f38c626, store: [table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:28,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/2308a93adff248cfad55b97c8f38c626 is 175, key is test_row_0/A:col10/1732153586724/Put/seqid=0 2024-11-21T01:46:28,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742473_1649 (size=31255) 2024-11-21T01:46:28,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-21T01:46:28,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:28,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153648223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:28,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:28,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:28,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:28,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153648224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:28,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:28,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:28,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:28,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:28,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:28,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153648224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:28,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:28,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:28,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:28,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153648224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:28,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:28,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:28,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:28,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:28,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:28,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153648224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:28,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:28,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:28,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:28,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153648532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:28,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:28,536 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:28,536 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153648533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:28,536 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:28,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:28,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:28,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153648533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:28,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:28,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:28,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:28,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:28,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153648533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:28,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:28,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:28,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153648533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:28,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:28,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:28,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:28,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:28,601 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=306, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/2308a93adff248cfad55b97c8f38c626 2024-11-21T01:46:28,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/3c566c9635f6428f959a8bf8bdc738d4 is 50, key is test_row_0/B:col10/1732153586724/Put/seqid=0 2024-11-21T01:46:28,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742474_1650 (size=12301) 2024-11-21T01:46:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-21T01:46:29,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153649043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:29,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:29,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:29,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:29,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153649043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:29,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153649043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:29,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153649043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:29,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:29,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:29,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:29,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:29,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:29,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:29,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:29,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153649044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:29,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:29,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:29,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:29,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:29,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:29,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:29,101 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/3c566c9635f6428f959a8bf8bdc738d4 2024-11-21T01:46:29,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/556131495b2a454696c8187101081fe3 is 50, key is test_row_0/C:col10/1732153586724/Put/seqid=0 2024-11-21T01:46:29,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742475_1651 (size=12301) 2024-11-21T01:46:29,145 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/556131495b2a454696c8187101081fe3 2024-11-21T01:46:29,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/2308a93adff248cfad55b97c8f38c626 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/2308a93adff248cfad55b97c8f38c626 2024-11-21T01:46:29,169 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/2308a93adff248cfad55b97c8f38c626, entries=150, sequenceid=306, filesize=30.5 K 2024-11-21T01:46:29,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/3c566c9635f6428f959a8bf8bdc738d4 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/3c566c9635f6428f959a8bf8bdc738d4 2024-11-21T01:46:29,174 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/3c566c9635f6428f959a8bf8bdc738d4, entries=150, sequenceid=306, filesize=12.0 K 2024-11-21T01:46:29,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/556131495b2a454696c8187101081fe3 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/556131495b2a454696c8187101081fe3 2024-11-21T01:46:29,180 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/556131495b2a454696c8187101081fe3, entries=150, sequenceid=306, filesize=12.0 K 2024-11-21T01:46:29,181 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f1185a7567ce5119c4e675690fbe9d2b in 1431ms, sequenceid=306, compaction requested=false 2024-11-21T01:46:29,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2603): Flush status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:29,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:29,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=175 2024-11-21T01:46:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=175 2024-11-21T01:46:29,186 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-21T01:46:29,186 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5860 sec 2024-11-21T01:46:29,189 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees in 1.5940 sec 2024-11-21T01:46:29,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-21T01:46:29,753 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:46:29,757 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:46:29,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees 2024-11-21T01:46:29,762 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=176, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:46:29,763 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=176, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:46:29,763 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=177, ppid=176, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:46:29,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-21T01:46:29,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-21T01:46:29,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-21T01:46:29,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:29,916 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2902): Flushing f1185a7567ce5119c4e675690fbe9d2b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T01:46:29,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=A 2024-11-21T01:46:29,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:29,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=B 2024-11-21T01:46:29,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:29,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=C 2024-11-21T01:46:29,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:29,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ff578c66d19846d2bc414f050b9bb8dd_f1185a7567ce5119c4e675690fbe9d2b is 50, key is test_row_0/A:col10/1732153587889/Put/seqid=0 2024-11-21T01:46:29,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742476_1652 (size=12454) 2024-11-21T01:46:29,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:29,949 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ff578c66d19846d2bc414f050b9bb8dd_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ff578c66d19846d2bc414f050b9bb8dd_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:29,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/99b638bd621846dbad6ebc2674f1cd3c, store: [table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:29,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/99b638bd621846dbad6ebc2674f1cd3c is 175, key is test_row_0/A:col10/1732153587889/Put/seqid=0 2024-11-21T01:46:29,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742477_1653 (size=31255) 2024-11-21T01:46:30,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:30,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:30,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-21T01:46:30,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:30,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153650106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:30,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732153590099, max timestamp=1732153590112Num uniques 859; 2024-11-21T01:46:30,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T01:46:30,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:30,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732153590099, max timestamp=1732153590112Num uniques 859; 2024-11-21T01:46:30,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-21T01:46:30,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732153590099, max timestamp=1732153590112Num uniques 859; 2024-11-21T01:46:30,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-21T01:46:30,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:30,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153650156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:30,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:30,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153650164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:30,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:30,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:30,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153650164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:30,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:30,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:30,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153650166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:30,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:30,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153650223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:30,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:30,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:30,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153650273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:30,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:30,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:30,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153650277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:30,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:30,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:30,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153650277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:30,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:30,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153650277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:30,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:30,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:30,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-21T01:46:30,393 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=321, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/99b638bd621846dbad6ebc2674f1cd3c 2024-11-21T01:46:30,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/2e19af4ece174fe6bd61ca8cbbde2a76 is 50, key is test_row_0/B:col10/1732153587889/Put/seqid=0 2024-11-21T01:46:30,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742478_1654 (size=12301) 2024-11-21T01:46:30,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:30,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153650443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:30,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:30,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:30,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153650483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:30,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:30,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153650483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:30,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:30,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:30,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:30,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153650485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:30,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:30,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:30,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153650493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:30,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:30,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:30,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153650753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:30,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:30,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:30,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153650793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:30,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:30,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:30,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153650793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:30,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:30,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:30,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153650803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:30,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:30,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:30,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153650803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:30,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:30,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:30,839 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/2e19af4ece174fe6bd61ca8cbbde2a76 2024-11-21T01:46:30,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/d3faa3f2391f4f428e7d8cc786e41247 is 50, key is test_row_0/C:col10/1732153587889/Put/seqid=0 2024-11-21T01:46:30,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742479_1655 (size=12301) 2024-11-21T01:46:30,859 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/d3faa3f2391f4f428e7d8cc786e41247 2024-11-21T01:46:30,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/99b638bd621846dbad6ebc2674f1cd3c as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/99b638bd621846dbad6ebc2674f1cd3c 2024-11-21T01:46:30,866 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/99b638bd621846dbad6ebc2674f1cd3c, entries=150, sequenceid=321, filesize=30.5 K 2024-11-21T01:46:30,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/2e19af4ece174fe6bd61ca8cbbde2a76 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/2e19af4ece174fe6bd61ca8cbbde2a76 2024-11-21T01:46:30,872 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/2e19af4ece174fe6bd61ca8cbbde2a76, entries=150, sequenceid=321, filesize=12.0 K 2024-11-21T01:46:30,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/d3faa3f2391f4f428e7d8cc786e41247 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/d3faa3f2391f4f428e7d8cc786e41247 2024-11-21T01:46:30,876 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/d3faa3f2391f4f428e7d8cc786e41247, entries=150, sequenceid=321, filesize=12.0 K 2024-11-21T01:46:30,877 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=135.63 KB/138882 for f1185a7567ce5119c4e675690fbe9d2b in 961ms, sequenceid=321, compaction requested=true 2024-11-21T01:46:30,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2603): Flush status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:30,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:30,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=177 2024-11-21T01:46:30,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=177 2024-11-21T01:46:30,880 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=177, resume processing ppid=176 2024-11-21T01:46:30,881 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=177, ppid=176, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1160 sec 2024-11-21T01:46:30,882 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees in 1.1240 sec 2024-11-21T01:46:30,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-21T01:46:30,903 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:46:30,904 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:46:30,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=178, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees 2024-11-21T01:46:30,906 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=178, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:46:30,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-21T01:46:30,907 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=178, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:46:30,907 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=179, ppid=178, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:46:31,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-21T01:46:31,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-21T01:46:31,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:31,062 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2902): Flushing f1185a7567ce5119c4e675690fbe9d2b 3/3 column families, dataSize=135.63 KB heapSize=388.23 KB 2024-11-21T01:46:31,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=A 2024-11-21T01:46:31,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:46:31,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=B 2024-11-21T01:46:31,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:46:31,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=C 2024-11-21T01:46:31,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:46:31,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216e0fdd4d81e3469fbd33757d0098e37a_f1185a7567ce5119c4e675690fbe9d2b is 50, key is test_row_0/A:col10/1732153590153/Put/seqid=0 2024-11-21T01:46:31,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742480_1656 (size=12454) 2024-11-21T01:46:31,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:31,098 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216e0fdd4d81e3469fbd33757d0098e37a_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216e0fdd4d81e3469fbd33757d0098e37a_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:31,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/89735f0146904533a1427340e0cc4601, store: [table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:31,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/89735f0146904533a1427340e0cc4601 is 175, key is test_row_0/A:col10/1732153590153/Put/seqid=0 2024-11-21T01:46:31,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742481_1657 (size=31255) 2024-11-21T01:46:31,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-21T01:46:31,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:31,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:31,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:31,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153651302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:31,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:31,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:31,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:31,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153651303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:31,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:31,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:31,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:31,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:31,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:31,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153651314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:31,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:31,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153651314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:31,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:31,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153651314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:31,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:31,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:31,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:31,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:31,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:31,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:31,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:31,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:31,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:31,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153651412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:31,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:31,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:31,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:31,515 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=358, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/89735f0146904533a1427340e0cc4601 2024-11-21T01:46:31,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/cb32703c3d8c4002890be12009ea8edc is 50, key is test_row_0/B:col10/1732153590153/Put/seqid=0 2024-11-21T01:46:31,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742482_1658 (size=12301) 2024-11-21T01:46:31,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-21T01:46:31,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153651623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:31,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:31,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:31,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:31,929 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/cb32703c3d8c4002890be12009ea8edc 2024-11-21T01:46:31,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/2f701a6a3b7e42e2b3758a29f4c2d58c is 50, key is test_row_0/C:col10/1732153590153/Put/seqid=0 2024-11-21T01:46:31,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742483_1659 (size=12301) 2024-11-21T01:46:31,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:31,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153651932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:31,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:31,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:31,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:32,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-21T01:46:32,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:32,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153652323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:32,326 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to b93ada835ef8,41129,1732153434359 for 'test_row_0' in f1185a7567ce5119c4e675690fbe9d2b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T01:46:32,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:32,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:32,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:32,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:32,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153652333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:32,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:32,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153652333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:32,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:32,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153652333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:32,337 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b93ada835ef8,41129,1732153434359 for 'test_row_0' in f1185a7567ce5119c4e675690fbe9d2b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T01:46:32,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:32,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:32,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:32,337 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b93ada835ef8,41129,1732153434359 for 'test_row_2' in f1185a7567ce5119c4e675690fbe9d2b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T01:46:32,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:32,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:32,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:32,338 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b93ada835ef8,41129,1732153434359 for 'test_row_0' in f1185a7567ce5119c4e675690fbe9d2b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T01:46:32,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:32,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:32,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:32,340 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/2f701a6a3b7e42e2b3758a29f4c2d58c 2024-11-21T01:46:32,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/89735f0146904533a1427340e0cc4601 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/89735f0146904533a1427340e0cc4601 2024-11-21T01:46:32,348 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/89735f0146904533a1427340e0cc4601, entries=150, sequenceid=358, filesize=30.5 K 2024-11-21T01:46:32,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/cb32703c3d8c4002890be12009ea8edc as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/cb32703c3d8c4002890be12009ea8edc 2024-11-21T01:46:32,354 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/cb32703c3d8c4002890be12009ea8edc, entries=150, sequenceid=358, filesize=12.0 K 2024-11-21T01:46:32,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/2f701a6a3b7e42e2b3758a29f4c2d58c as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/2f701a6a3b7e42e2b3758a29f4c2d58c 2024-11-21T01:46:32,359 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/2f701a6a3b7e42e2b3758a29f4c2d58c, entries=150, sequenceid=358, filesize=12.0 K 2024-11-21T01:46:32,360 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(3140): Finished flush of dataSize ~135.63 KB/138882, heapSize ~388.19 KB/397504, currentSize=46.96 KB/48090 for f1185a7567ce5119c4e675690fbe9d2b in 1298ms, sequenceid=358, compaction requested=true 2024-11-21T01:46:32,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2603): Flush status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:32,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:32,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=179 2024-11-21T01:46:32,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=179 2024-11-21T01:46:32,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=179, resume processing ppid=178 2024-11-21T01:46:32,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=179, ppid=178, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4540 sec 2024-11-21T01:46:32,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=178, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees in 1.4620 sec 2024-11-21T01:46:32,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:32,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f1185a7567ce5119c4e675690fbe9d2b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-21T01:46:32,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=A 2024-11-21T01:46:32,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:32,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=B 2024-11-21T01:46:32,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:32,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=C 2024-11-21T01:46:32,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:32,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112158e0556bd4a841e39f744f333364b244_f1185a7567ce5119c4e675690fbe9d2b is 50, key is test_row_0/A:col10/1732153591295/Put/seqid=0 2024-11-21T01:46:32,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742484_1660 (size=12454) 2024-11-21T01:46:32,478 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:32,486 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112158e0556bd4a841e39f744f333364b244_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112158e0556bd4a841e39f744f333364b244_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:32,486 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/0e9b9512eb46400e8fa0442ab21a5ec1, store: [table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:32,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/0e9b9512eb46400e8fa0442ab21a5ec1 is 175, key is test_row_0/A:col10/1732153591295/Put/seqid=0 2024-11-21T01:46:32,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742485_1661 (size=31255) 2024-11-21T01:46:32,493 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=369, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/0e9b9512eb46400e8fa0442ab21a5ec1 2024-11-21T01:46:32,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/dafef24ba08243e888abdec6c89d1357 is 50, key is test_row_0/B:col10/1732153591295/Put/seqid=0 2024-11-21T01:46:32,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742486_1662 (size=12301) 2024-11-21T01:46:32,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732153592545, max timestamp=1732153592563Num uniques 859; 2024-11-21T01:46:32,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T01:46:32,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732153592545, max timestamp=1732153592563Num uniques 859; 2024-11-21T01:46:32,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T01:46:32,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732153592545, max timestamp=1732153592563Num uniques 859; 2024-11-21T01:46:32,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b93ada835ef8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T01:46:32,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:32,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153652671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:32,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:32,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:32,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:32,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:32,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153652782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:32,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:32,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:32,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:32,906 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/dafef24ba08243e888abdec6c89d1357 2024-11-21T01:46:32,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/94634762b21048d2bb289bf50a20fe0e is 50, key is test_row_0/C:col10/1732153591295/Put/seqid=0 2024-11-21T01:46:32,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742487_1663 (size=12301) 2024-11-21T01:46:32,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/94634762b21048d2bb289bf50a20fe0e 2024-11-21T01:46:32,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/0e9b9512eb46400e8fa0442ab21a5ec1 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/0e9b9512eb46400e8fa0442ab21a5ec1 2024-11-21T01:46:32,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/0e9b9512eb46400e8fa0442ab21a5ec1, entries=150, sequenceid=369, filesize=30.5 K 2024-11-21T01:46:32,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/dafef24ba08243e888abdec6c89d1357 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/dafef24ba08243e888abdec6c89d1357 2024-11-21T01:46:32,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/dafef24ba08243e888abdec6c89d1357, entries=150, sequenceid=369, filesize=12.0 K 2024-11-21T01:46:32,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/94634762b21048d2bb289bf50a20fe0e as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/94634762b21048d2bb289bf50a20fe0e 2024-11-21T01:46:32,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/94634762b21048d2bb289bf50a20fe0e, entries=150, sequenceid=369, filesize=12.0 K 2024-11-21T01:46:32,958 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for f1185a7567ce5119c4e675690fbe9d2b in 501ms, sequenceid=369, compaction requested=true 2024-11-21T01:46:32,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:32,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f1185a7567ce5119c4e675690fbe9d2b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T01:46:32,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:32,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f1185a7567ce5119c4e675690fbe9d2b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T01:46:32,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:32,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f1185a7567ce5119c4e675690fbe9d2b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T01:46:32,958 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-21T01:46:32,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:46:32,958 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-21T01:46:32,959 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62153 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-21T01:46:32,959 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): f1185a7567ce5119c4e675690fbe9d2b/B is initiating minor compaction (all files) 2024-11-21T01:46:32,959 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f1185a7567ce5119c4e675690fbe9d2b/B in TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:32,959 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/3702d53c68ce4dc0be534f1c0b376447, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/3c566c9635f6428f959a8bf8bdc738d4, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/2e19af4ece174fe6bd61ca8cbbde2a76, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/cb32703c3d8c4002890be12009ea8edc, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/dafef24ba08243e888abdec6c89d1357] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp, totalSize=60.7 K 2024-11-21T01:46:32,960 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 156923 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-21T01:46:32,960 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1541): f1185a7567ce5119c4e675690fbe9d2b/A is initiating minor compaction (all files) 2024-11-21T01:46:32,960 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f1185a7567ce5119c4e675690fbe9d2b/A in TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:32,960 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/562af2bd4984438d8fdba0d36d9b13d9, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/2308a93adff248cfad55b97c8f38c626, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/99b638bd621846dbad6ebc2674f1cd3c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/89735f0146904533a1427340e0cc4601, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/0e9b9512eb46400e8fa0442ab21a5ec1] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp, totalSize=153.2 K 2024-11-21T01:46:32,960 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:32,960 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. files: [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/562af2bd4984438d8fdba0d36d9b13d9, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/2308a93adff248cfad55b97c8f38c626, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/99b638bd621846dbad6ebc2674f1cd3c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/89735f0146904533a1427340e0cc4601, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/0e9b9512eb46400e8fa0442ab21a5ec1] 2024-11-21T01:46:32,961 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 3702d53c68ce4dc0be534f1c0b376447, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732153584472 2024-11-21T01:46:32,961 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 562af2bd4984438d8fdba0d36d9b13d9, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732153584472 2024-11-21T01:46:32,961 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c566c9635f6428f959a8bf8bdc738d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732153586713 2024-11-21T01:46:32,961 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2308a93adff248cfad55b97c8f38c626, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732153586713 2024-11-21T01:46:32,961 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e19af4ece174fe6bd61ca8cbbde2a76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732153587889 2024-11-21T01:46:32,961 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99b638bd621846dbad6ebc2674f1cd3c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732153587889 2024-11-21T01:46:32,962 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting cb32703c3d8c4002890be12009ea8edc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732153590145 2024-11-21T01:46:32,962 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89735f0146904533a1427340e0cc4601, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732153590145 2024-11-21T01:46:32,962 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting dafef24ba08243e888abdec6c89d1357, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732153591295 2024-11-21T01:46:32,962 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e9b9512eb46400e8fa0442ab21a5ec1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732153591295 2024-11-21T01:46:32,973 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f1185a7567ce5119c4e675690fbe9d2b#B#compaction#561 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:32,973 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/ab7891097c5d4136b1fa1536f95dcfa1 is 50, key is test_row_0/B:col10/1732153591295/Put/seqid=0 2024-11-21T01:46:32,976 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:32,979 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112119fee50bdcc64e2e81b1765038628d52_f1185a7567ce5119c4e675690fbe9d2b store=[table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:32,982 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112119fee50bdcc64e2e81b1765038628d52_f1185a7567ce5119c4e675690fbe9d2b, store=[table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:32,983 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112119fee50bdcc64e2e81b1765038628d52_f1185a7567ce5119c4e675690fbe9d2b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:32,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742488_1664 (size=13119) 2024-11-21T01:46:32,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:32,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f1185a7567ce5119c4e675690fbe9d2b 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-21T01:46:32,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=A 2024-11-21T01:46:32,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:46:32,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=B 2024-11-21T01:46:32,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:46:32,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=C 2024-11-21T01:46:32,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T01:46:32,997 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/ab7891097c5d4136b1fa1536f95dcfa1 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/ab7891097c5d4136b1fa1536f95dcfa1 2024-11-21T01:46:33,003 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f1185a7567ce5119c4e675690fbe9d2b/B of f1185a7567ce5119c4e675690fbe9d2b into ab7891097c5d4136b1fa1536f95dcfa1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:33,003 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:33,003 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., storeName=f1185a7567ce5119c4e675690fbe9d2b/B, priority=11, startTime=1732153592958; duration=0sec 2024-11-21T01:46:33,003 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T01:46:33,003 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f1185a7567ce5119c4e675690fbe9d2b:B 2024-11-21T01:46:33,003 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-21T01:46:33,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742489_1665 (size=4469) 2024-11-21T01:46:33,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d047de01238743689dfb505a9bd4771e_f1185a7567ce5119c4e675690fbe9d2b is 50, key is test_row_0/A:col10/1732153592653/Put/seqid=0 2024-11-21T01:46:33,009 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62153 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-21T01:46:33,009 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f1185a7567ce5119c4e675690fbe9d2b#A#compaction#562 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:33,009 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1541): f1185a7567ce5119c4e675690fbe9d2b/C is initiating minor compaction (all files) 2024-11-21T01:46:33,009 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f1185a7567ce5119c4e675690fbe9d2b/C in TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:33,009 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/dfb37d1712b84f378505ca723a3c15a3, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/556131495b2a454696c8187101081fe3, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/d3faa3f2391f4f428e7d8cc786e41247, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/2f701a6a3b7e42e2b3758a29f4c2d58c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/94634762b21048d2bb289bf50a20fe0e] into tmpdir=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp, totalSize=60.7 K 2024-11-21T01:46:33,009 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/b1b42505396d4c569cc93a29a8b43b7e is 175, key is test_row_0/A:col10/1732153591295/Put/seqid=0 2024-11-21T01:46:33,009 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting dfb37d1712b84f378505ca723a3c15a3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732153584472 2024-11-21T01:46:33,010 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 556131495b2a454696c8187101081fe3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732153586713 2024-11-21T01:46:33,011 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting d3faa3f2391f4f428e7d8cc786e41247, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732153587889 2024-11-21T01:46:33,011 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f701a6a3b7e42e2b3758a29f4c2d58c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732153590145 2024-11-21T01:46:33,011 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] compactions.Compactor(225): Compacting 94634762b21048d2bb289bf50a20fe0e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732153591295 2024-11-21T01:46:33,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742490_1666 (size=14994) 2024-11-21T01:46:33,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742491_1667 (size=32073) 2024-11-21T01:46:33,021 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f1185a7567ce5119c4e675690fbe9d2b#C#compaction#564 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T01:46:33,021 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/1b6e717383c64e99a4b6917a28421b2a is 50, key is test_row_0/C:col10/1732153591295/Put/seqid=0 2024-11-21T01:46:33,025 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/b1b42505396d4c569cc93a29a8b43b7e as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/b1b42505396d4c569cc93a29a8b43b7e 2024-11-21T01:46:33,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742492_1668 (size=13119) 2024-11-21T01:46:33,030 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f1185a7567ce5119c4e675690fbe9d2b/A of f1185a7567ce5119c4e675690fbe9d2b into b1b42505396d4c569cc93a29a8b43b7e(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:33,030 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:33,030 INFO [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., storeName=f1185a7567ce5119c4e675690fbe9d2b/A, priority=11, startTime=1732153592958; duration=0sec 2024-11-21T01:46:33,030 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:33,030 DEBUG [RS:0;b93ada835ef8:41129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f1185a7567ce5119c4e675690fbe9d2b:A 2024-11-21T01:46:33,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:33,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153653045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:33,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:33,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:33,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:33,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-21T01:46:33,053 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:46:33,053 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-21T01:46:33,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=180, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees 2024-11-21T01:46:33,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-21T01:46:33,055 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=180, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T01:46:33,056 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=180, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T01:46:33,056 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=181, ppid=180, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T01:46:33,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:33,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153653153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:33,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:33,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:33,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:33,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-21T01:46:33,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-11-21T01:46:33,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:33,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:33,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:33,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:33,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:33,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:33,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-11-21T01:46:33,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:33,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:33,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:33,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:33,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:33,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:33,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:33,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153653365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:33,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:33,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:33,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:33,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-21T01:46:33,416 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:33,423 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d047de01238743689dfb505a9bd4771e_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d047de01238743689dfb505a9bd4771e_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:33,424 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/5c54706fe72844c5b8bcf3fc5d288609, store: [table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:33,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/5c54706fe72844c5b8bcf3fc5d288609 is 175, key is test_row_0/A:col10/1732153592653/Put/seqid=0 2024-11-21T01:46:33,431 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/1b6e717383c64e99a4b6917a28421b2a as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/1b6e717383c64e99a4b6917a28421b2a 2024-11-21T01:46:33,435 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f1185a7567ce5119c4e675690fbe9d2b/C of f1185a7567ce5119c4e675690fbe9d2b into 1b6e717383c64e99a4b6917a28421b2a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T01:46:33,435 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:33,435 INFO [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., storeName=f1185a7567ce5119c4e675690fbe9d2b/C, priority=11, startTime=1732153592958; duration=0sec 2024-11-21T01:46:33,436 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T01:46:33,436 DEBUG [RS:0;b93ada835ef8:41129-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f1185a7567ce5119c4e675690fbe9d2b:C 2024-11-21T01:46:33,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742493_1669 (size=39949) 2024-11-21T01:46:33,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-11-21T01:46:33,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:33,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:33,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:33,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:33,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:33,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:33,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-11-21T01:46:33,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:33,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:33,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:33,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:33,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:33,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:33,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153653673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:33,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:33,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:33,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:33,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:33,680 INFO [Thread-2526 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2526. 2024-11-21T01:46:33,680 DEBUG [Thread-2526 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:46:33,680 DEBUG [Thread-2526 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:33,680 DEBUG [Thread-2526 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:33,680 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:46:33,681 INFO [Thread-2522 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2522. 2024-11-21T01:46:33,681 DEBUG [Thread-2522 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:46:33,681 DEBUG [Thread-2522 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:33,681 DEBUG [Thread-2522 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:33,681 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:46:33,682 INFO [Thread-2524 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2524. 2024-11-21T01:46:33,682 DEBUG [Thread-2524 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:46:33,682 DEBUG [Thread-2524 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:33,682 DEBUG [Thread-2524 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:33,682 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:46:33,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-21T01:46:33,685 INFO [Thread-2518 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2518. 2024-11-21T01:46:33,685 DEBUG [Thread-2518 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:46:33,685 DEBUG [Thread-2518 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:33,685 DEBUG [Thread-2518 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:33,685 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:46:33,686 INFO [Thread-2520 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2520. 2024-11-21T01:46:33,686 DEBUG [Thread-2520 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:46:33,686 DEBUG [Thread-2520 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:33,686 DEBUG [Thread-2520 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:33,686 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:46:33,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-11-21T01:46:33,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:33,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:33,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:33,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:33,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:33,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:33,838 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=406, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/5c54706fe72844c5b8bcf3fc5d288609 2024-11-21T01:46:33,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/2a101083cbd24d678349efd7a170445a is 50, key is test_row_0/B:col10/1732153592653/Put/seqid=0 2024-11-21T01:46:33,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742494_1670 (size=12301) 2024-11-21T01:46:33,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-11-21T01:46:33,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:33,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:33,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:33,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:33,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:33,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:34,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-11-21T01:46:34,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:34,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:34,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:34,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:34,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:34,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:34,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:34,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48310 deadline: 1732153654182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:34,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:34,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:34,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:34,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-21T01:46:34,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/2a101083cbd24d678349efd7a170445a 2024-11-21T01:46:34,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/049adb772c5c489db92c8f29512c4f62 is 50, key is test_row_0/C:col10/1732153592653/Put/seqid=0 2024-11-21T01:46:34,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742495_1671 (size=12301) 2024-11-21T01:46:34,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-11-21T01:46:34,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:34,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:34,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:34,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:34,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:34,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:34,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:34,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48324 deadline: 1732153654333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:34,333 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to b93ada835ef8,41129,1732153434359 for 'test_row_0' in f1185a7567ce5119c4e675690fbe9d2b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T01:46:34,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:34,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:34,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:34,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:34,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732153654353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:34,353 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b93ada835ef8,41129,1732153434359 for 'test_row_0' in f1185a7567ce5119c4e675690fbe9d2b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T01:46:34,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:34,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:34,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:34,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:34,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48290 deadline: 1732153654363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:34,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:46:34,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48266 deadline: 1732153654363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:34,364 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b93ada835ef8,41129,1732153434359 for 'test_row_2' in f1185a7567ce5119c4e675690fbe9d2b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T01:46:34,364 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b93ada835ef8,41129,1732153434359 for 'test_row_0' in f1185a7567ce5119c4e675690fbe9d2b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T01:46:34,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:34,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:34,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 , the old value is region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:34,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:34,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T01:46:34,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., hostname=b93ada835ef8,41129,1732153434359, seqNum=5 because the exception is null or not the one we care about 2024-11-21T01:46:34,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-11-21T01:46:34,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:34,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:34,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:34,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:34,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:34,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:34,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-11-21T01:46:34,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:34,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:34,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:34,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:34,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:34,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:46:34,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/049adb772c5c489db92c8f29512c4f62 2024-11-21T01:46:34,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/5c54706fe72844c5b8bcf3fc5d288609 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/5c54706fe72844c5b8bcf3fc5d288609 2024-11-21T01:46:34,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/5c54706fe72844c5b8bcf3fc5d288609, entries=200, sequenceid=406, filesize=39.0 K 2024-11-21T01:46:34,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/2a101083cbd24d678349efd7a170445a as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/2a101083cbd24d678349efd7a170445a 2024-11-21T01:46:34,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/2a101083cbd24d678349efd7a170445a, entries=150, sequenceid=406, filesize=12.0 K 2024-11-21T01:46:34,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/049adb772c5c489db92c8f29512c4f62 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/049adb772c5c489db92c8f29512c4f62 2024-11-21T01:46:34,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/049adb772c5c489db92c8f29512c4f62, entries=150, sequenceid=406, filesize=12.0 K 2024-11-21T01:46:34,670 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for f1185a7567ce5119c4e675690fbe9d2b in 1674ms, sequenceid=406, compaction requested=false 2024-11-21T01:46:34,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:34,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-11-21T01:46:34,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:34,743 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2902): Flushing f1185a7567ce5119c4e675690fbe9d2b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T01:46:34,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=A 2024-11-21T01:46:34,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:34,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=B 2024-11-21T01:46:34,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:34,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=C 2024-11-21T01:46:34,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:34,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d8d5928ed63944b08b7c005c6c97d3c2_f1185a7567ce5119c4e675690fbe9d2b is 50, key is test_row_0/A:col10/1732153593024/Put/seqid=0 2024-11-21T01:46:34,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742496_1672 (size=12454) 2024-11-21T01:46:35,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:35,154 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d8d5928ed63944b08b7c005c6c97d3c2_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d8d5928ed63944b08b7c005c6c97d3c2_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:35,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/14693c634d654681b008f1bdace2d6aa, store: [table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:35,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/14693c634d654681b008f1bdace2d6aa is 175, key is test_row_0/A:col10/1732153593024/Put/seqid=0 2024-11-21T01:46:35,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742497_1673 (size=31255) 2024-11-21T01:46:35,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41129 {}] regionserver.HRegion(8855): Flush requested on f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:35,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. as already flushing 2024-11-21T01:46:35,194 INFO [Thread-2513 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2513. 2024-11-21T01:46:35,194 DEBUG [Thread-2513 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:46:35,194 DEBUG [Thread-2513 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:35,194 DEBUG [Thread-2513 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:35,194 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:46:35,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-21T01:46:35,559 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=419, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/14693c634d654681b008f1bdace2d6aa 2024-11-21T01:46:35,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/0db9c77609e04df5b92cb4e2dfba2c22 is 50, key is test_row_0/B:col10/1732153593024/Put/seqid=0 2024-11-21T01:46:35,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742498_1674 (size=12301) 2024-11-21T01:46:35,970 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/0db9c77609e04df5b92cb4e2dfba2c22 2024-11-21T01:46:35,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/9296a75f0e8542a9b3ea27725cef9c61 is 50, key is test_row_0/C:col10/1732153593024/Put/seqid=0 2024-11-21T01:46:35,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742499_1675 (size=12301) 2024-11-21T01:46:36,378 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/9296a75f0e8542a9b3ea27725cef9c61 2024-11-21T01:46:36,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/14693c634d654681b008f1bdace2d6aa as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/14693c634d654681b008f1bdace2d6aa 2024-11-21T01:46:36,385 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/14693c634d654681b008f1bdace2d6aa, entries=150, sequenceid=419, filesize=30.5 K 2024-11-21T01:46:36,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/0db9c77609e04df5b92cb4e2dfba2c22 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/0db9c77609e04df5b92cb4e2dfba2c22 2024-11-21T01:46:36,388 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/0db9c77609e04df5b92cb4e2dfba2c22, entries=150, sequenceid=419, filesize=12.0 K 2024-11-21T01:46:36,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/9296a75f0e8542a9b3ea27725cef9c61 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/9296a75f0e8542a9b3ea27725cef9c61 2024-11-21T01:46:36,391 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/9296a75f0e8542a9b3ea27725cef9c61, entries=150, sequenceid=419, filesize=12.0 K 2024-11-21T01:46:36,392 INFO [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=6.71 KB/6870 for f1185a7567ce5119c4e675690fbe9d2b in 1649ms, sequenceid=419, compaction requested=true 2024-11-21T01:46:36,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2603): Flush status journal for f1185a7567ce5119c4e675690fbe9d2b: 2024-11-21T01:46:36,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:36,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=181 2024-11-21T01:46:36,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster(4169): Remote procedure done, pid=181 2024-11-21T01:46:36,395 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=181, resume processing ppid=180 2024-11-21T01:46:36,395 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=181, ppid=180, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3380 sec 2024-11-21T01:46:36,397 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=180, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees in 3.3420 sec 2024-11-21T01:46:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-21T01:46:37,223 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:46:38,354 INFO [Thread-2515 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2515. 2024-11-21T01:46:38,354 DEBUG [Thread-2515 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:46:38,354 DEBUG [Thread-2515 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:38,355 DEBUG [Thread-2515 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:38,355 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:46:38,385 INFO [Thread-2507 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2507. 2024-11-21T01:46:38,385 DEBUG [Thread-2507 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:46:38,385 DEBUG [Thread-2507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:38,386 DEBUG [Thread-2507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:38,386 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:46:38,395 INFO [Thread-2509 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2509. 2024-11-21T01:46:38,395 DEBUG [Thread-2509 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:46:38,395 DEBUG [Thread-2509 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:38,395 DEBUG [Thread-2509 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:38,395 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:46:38,415 INFO [Thread-2511 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2511. 2024-11-21T01:46:38,415 DEBUG [Thread-2511 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T01:46:38,415 DEBUG [Thread-2511 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:38,415 DEBUG [Thread-2511 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:38,415 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:46:38,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T01:46:38,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-21T01:46:38,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-21T01:46:38,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-21T01:46:38,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-11-21T01:46:38,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-21T01:46:38,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T01:46:38,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2599 2024-11-21T01:46:38,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2495 2024-11-21T01:46:38,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2466 2024-11-21T01:46:38,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2625 2024-11-21T01:46:38,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2462 2024-11-21T01:46:38,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T01:46:38,415 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T01:46:38,415 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T01:46:38,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:38,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:38,415 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:46:38,416 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-21T01:46:38,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=182, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T01:46:38,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-21T01:46:38,419 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732153598419"}]},"ts":"1732153598419"} 2024-11-21T01:46:38,420 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T01:46:38,421 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T01:46:38,421 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=183, ppid=182, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T01:46:38,423 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=184, ppid=183, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f1185a7567ce5119c4e675690fbe9d2b, UNASSIGN}] 2024-11-21T01:46:38,423 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=184, ppid=183, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f1185a7567ce5119c4e675690fbe9d2b, UNASSIGN 2024-11-21T01:46:38,424 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=184 updating hbase:meta row=f1185a7567ce5119c4e675690fbe9d2b, regionState=CLOSING, regionLocation=b93ada835ef8,41129,1732153434359 2024-11-21T01:46:38,425 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=184, ppid=183, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f1185a7567ce5119c4e675690fbe9d2b, UNASSIGN because future has completed 2024-11-21T01:46:38,426 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T01:46:38,426 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=185, ppid=184, state=RUNNABLE, hasLock=false; CloseRegionProcedure f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359}] 2024-11-21T01:46:38,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-21T01:46:38,578 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] handler.UnassignRegionHandler(122): Close f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:38,578 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T01:46:38,578 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(1722): Closing f1185a7567ce5119c4e675690fbe9d2b, disabling compactions & flushes 2024-11-21T01:46:38,578 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:38,578 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:38,578 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. after waiting 0 ms 2024-11-21T01:46:38,578 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:38,578 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(2902): Flushing f1185a7567ce5119c4e675690fbe9d2b 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-21T01:46:38,578 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=A 2024-11-21T01:46:38,579 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:38,579 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=B 2024-11-21T01:46:38,579 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:38,579 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f1185a7567ce5119c4e675690fbe9d2b, store=C 2024-11-21T01:46:38,579 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T01:46:38,583 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f38e0016044c4d1c85f1d7a167a95b0f_f1185a7567ce5119c4e675690fbe9d2b is 50, key is test_row_0/A:col10/1732153598393/Put/seqid=0 2024-11-21T01:46:38,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742500_1676 (size=12454) 2024-11-21T01:46:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-21T01:46:38,986 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:46:38,990 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f38e0016044c4d1c85f1d7a167a95b0f_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f38e0016044c4d1c85f1d7a167a95b0f_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:38,990 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/9f2087028b9443cfb6a23cd52e0254e8, store: [table=TestAcidGuarantees family=A region=f1185a7567ce5119c4e675690fbe9d2b] 2024-11-21T01:46:38,991 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/9f2087028b9443cfb6a23cd52e0254e8 is 175, key is test_row_0/A:col10/1732153598393/Put/seqid=0 2024-11-21T01:46:38,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742501_1677 (size=31255) 2024-11-21T01:46:39,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-21T01:46:39,395 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=427, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/9f2087028b9443cfb6a23cd52e0254e8 2024-11-21T01:46:39,400 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/1567410e67704b589b1d94dedb746c0a is 50, key is test_row_0/B:col10/1732153598393/Put/seqid=0 2024-11-21T01:46:39,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742502_1678 (size=12301) 2024-11-21T01:46:39,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-21T01:46:39,804 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/1567410e67704b589b1d94dedb746c0a 2024-11-21T01:46:39,809 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/fdd5999231544e17acdbfb96cb9f0af3 is 50, key is test_row_0/C:col10/1732153598393/Put/seqid=0 2024-11-21T01:46:39,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742503_1679 (size=12301) 2024-11-21T01:46:40,213 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/fdd5999231544e17acdbfb96cb9f0af3 2024-11-21T01:46:40,216 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/A/9f2087028b9443cfb6a23cd52e0254e8 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/9f2087028b9443cfb6a23cd52e0254e8 2024-11-21T01:46:40,220 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/9f2087028b9443cfb6a23cd52e0254e8, entries=150, sequenceid=427, filesize=30.5 K 2024-11-21T01:46:40,220 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/B/1567410e67704b589b1d94dedb746c0a as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/1567410e67704b589b1d94dedb746c0a 2024-11-21T01:46:40,224 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/1567410e67704b589b1d94dedb746c0a, entries=150, sequenceid=427, filesize=12.0 K 2024-11-21T01:46:40,224 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/.tmp/C/fdd5999231544e17acdbfb96cb9f0af3 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/fdd5999231544e17acdbfb96cb9f0af3 2024-11-21T01:46:40,227 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/fdd5999231544e17acdbfb96cb9f0af3, entries=150, sequenceid=427, filesize=12.0 K 2024-11-21T01:46:40,228 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f1185a7567ce5119c4e675690fbe9d2b in 1649ms, sequenceid=427, compaction requested=true 2024-11-21T01:46:40,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/c1fa2a8a785445669e72f02e9581f560, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/bfb44d5463ed45168fd38576a37f13c8, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/807e33901df04b5794f47f2db3fe554f, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/2cf48b36feca49d1a5d69212cc5d78e8, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/55a6b6dbdc484698ae5800ec0dca1dda, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/c7fb1619a98643138da84776082d5de3, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/1a4f602decfc47bb992827c7435e5eac, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/b7680dcff0e445c9b96f1288b552c993, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/c3fa883a5386493bb1b6571e2fb4db47, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/29b46d3d15794e3a9035fcd0388e5d32, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/4d2036a0977e4d4b830f7ea04cb720f1, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/838a58087ef840099a1a696ec642b10d, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/43ea0c3865914ae4b092ec1d6b17952c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/b34ac67e0c304a7a8a3f556e3e05aba9, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/3bf9800d6fd5416ea669e1103fbc7884, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/804c6ee2e9714b3db36fe2bbe94694c8, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/bea94f6e7f114d66b4a13dba475689f2, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/09e0b535ff854c038c66d5fb0f8d87da, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/41670f50f23748efaa2a358c8c1cf1c3, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/562af2bd4984438d8fdba0d36d9b13d9, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/2308a93adff248cfad55b97c8f38c626, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/99b638bd621846dbad6ebc2674f1cd3c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/89735f0146904533a1427340e0cc4601, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/0e9b9512eb46400e8fa0442ab21a5ec1] to archive 2024-11-21T01:46:40,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T01:46:40,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/c1fa2a8a785445669e72f02e9581f560 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/c1fa2a8a785445669e72f02e9581f560 2024-11-21T01:46:40,231 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/bfb44d5463ed45168fd38576a37f13c8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/bfb44d5463ed45168fd38576a37f13c8 2024-11-21T01:46:40,231 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/807e33901df04b5794f47f2db3fe554f to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/807e33901df04b5794f47f2db3fe554f 2024-11-21T01:46:40,232 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/2cf48b36feca49d1a5d69212cc5d78e8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/2cf48b36feca49d1a5d69212cc5d78e8 2024-11-21T01:46:40,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/55a6b6dbdc484698ae5800ec0dca1dda to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/55a6b6dbdc484698ae5800ec0dca1dda 2024-11-21T01:46:40,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/c7fb1619a98643138da84776082d5de3 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/c7fb1619a98643138da84776082d5de3 2024-11-21T01:46:40,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/1a4f602decfc47bb992827c7435e5eac to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/1a4f602decfc47bb992827c7435e5eac 2024-11-21T01:46:40,235 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/b7680dcff0e445c9b96f1288b552c993 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/b7680dcff0e445c9b96f1288b552c993 2024-11-21T01:46:40,236 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/c3fa883a5386493bb1b6571e2fb4db47 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/c3fa883a5386493bb1b6571e2fb4db47 2024-11-21T01:46:40,236 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/29b46d3d15794e3a9035fcd0388e5d32 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/29b46d3d15794e3a9035fcd0388e5d32 2024-11-21T01:46:40,237 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/4d2036a0977e4d4b830f7ea04cb720f1 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/4d2036a0977e4d4b830f7ea04cb720f1 2024-11-21T01:46:40,238 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/838a58087ef840099a1a696ec642b10d to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/838a58087ef840099a1a696ec642b10d 2024-11-21T01:46:40,239 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/43ea0c3865914ae4b092ec1d6b17952c to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/43ea0c3865914ae4b092ec1d6b17952c 2024-11-21T01:46:40,240 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/b34ac67e0c304a7a8a3f556e3e05aba9 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/b34ac67e0c304a7a8a3f556e3e05aba9 2024-11-21T01:46:40,241 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/3bf9800d6fd5416ea669e1103fbc7884 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/3bf9800d6fd5416ea669e1103fbc7884 2024-11-21T01:46:40,242 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/804c6ee2e9714b3db36fe2bbe94694c8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/804c6ee2e9714b3db36fe2bbe94694c8 2024-11-21T01:46:40,243 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/bea94f6e7f114d66b4a13dba475689f2 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/bea94f6e7f114d66b4a13dba475689f2 2024-11-21T01:46:40,244 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/09e0b535ff854c038c66d5fb0f8d87da to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/09e0b535ff854c038c66d5fb0f8d87da 2024-11-21T01:46:40,245 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/41670f50f23748efaa2a358c8c1cf1c3 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/41670f50f23748efaa2a358c8c1cf1c3 2024-11-21T01:46:40,246 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/562af2bd4984438d8fdba0d36d9b13d9 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/562af2bd4984438d8fdba0d36d9b13d9 2024-11-21T01:46:40,247 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/2308a93adff248cfad55b97c8f38c626 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/2308a93adff248cfad55b97c8f38c626 2024-11-21T01:46:40,248 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/99b638bd621846dbad6ebc2674f1cd3c to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/99b638bd621846dbad6ebc2674f1cd3c 2024-11-21T01:46:40,249 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/89735f0146904533a1427340e0cc4601 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/89735f0146904533a1427340e0cc4601 2024-11-21T01:46:40,250 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/0e9b9512eb46400e8fa0442ab21a5ec1 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/0e9b9512eb46400e8fa0442ab21a5ec1 2024-11-21T01:46:40,251 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/0ff599c220c04b49adbd3a10ef355815, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/08de650fbfdc4e408ea92fc821fb4908, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/73db08aa8bc04b37bd50ec11eb19c6dc, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/c4c0192df5244a4ca6ff96919ed79625, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/70ab50c871bf49619554d723ce72842b, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/234a6f0c1fb843388278d844dd42726d, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/5fc8aa0965054a8ca6d74add971a518c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/317de57286894d3dac7dd2eca9233857, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/5eea9426c370417cb843860ad4b13726, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/a7b7b4daa5684c078721a2c673af545a, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/dea5bce0edbc48129338f3fe0f672f30, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/8338fce7cddd4eb7804109ef5dc4fff5, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/e076d21e822a43d9853dfa57a91d757c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/56ac2f2f78c844d8ad74b87f9d923068, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/1641efd28c18412d908e4d51ef3d4f59, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/f68223045ee54355a0366b7ab35174a0, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/5e4ed402d40a427aaad3b429c4d6daee, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/ded45e31221c40419d1d252b5c6b0dc2, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/3702d53c68ce4dc0be534f1c0b376447, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/3a87aed54d794522ad0ad9c264a497f3, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/3c566c9635f6428f959a8bf8bdc738d4, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/2e19af4ece174fe6bd61ca8cbbde2a76, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/cb32703c3d8c4002890be12009ea8edc, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/dafef24ba08243e888abdec6c89d1357] to archive 2024-11-21T01:46:40,252 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T01:46:40,252 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/0ff599c220c04b49adbd3a10ef355815 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/0ff599c220c04b49adbd3a10ef355815 2024-11-21T01:46:40,253 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/08de650fbfdc4e408ea92fc821fb4908 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/08de650fbfdc4e408ea92fc821fb4908 2024-11-21T01:46:40,254 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/73db08aa8bc04b37bd50ec11eb19c6dc to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/73db08aa8bc04b37bd50ec11eb19c6dc 2024-11-21T01:46:40,255 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/c4c0192df5244a4ca6ff96919ed79625 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/c4c0192df5244a4ca6ff96919ed79625 2024-11-21T01:46:40,255 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/70ab50c871bf49619554d723ce72842b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/70ab50c871bf49619554d723ce72842b 2024-11-21T01:46:40,256 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/234a6f0c1fb843388278d844dd42726d to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/234a6f0c1fb843388278d844dd42726d 2024-11-21T01:46:40,257 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/5fc8aa0965054a8ca6d74add971a518c to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/5fc8aa0965054a8ca6d74add971a518c 2024-11-21T01:46:40,258 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/317de57286894d3dac7dd2eca9233857 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/317de57286894d3dac7dd2eca9233857 2024-11-21T01:46:40,259 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/5eea9426c370417cb843860ad4b13726 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/5eea9426c370417cb843860ad4b13726 2024-11-21T01:46:40,259 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/a7b7b4daa5684c078721a2c673af545a to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/a7b7b4daa5684c078721a2c673af545a 2024-11-21T01:46:40,260 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/dea5bce0edbc48129338f3fe0f672f30 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/dea5bce0edbc48129338f3fe0f672f30 2024-11-21T01:46:40,261 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/8338fce7cddd4eb7804109ef5dc4fff5 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/8338fce7cddd4eb7804109ef5dc4fff5 2024-11-21T01:46:40,262 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/e076d21e822a43d9853dfa57a91d757c to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/e076d21e822a43d9853dfa57a91d757c 2024-11-21T01:46:40,262 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/56ac2f2f78c844d8ad74b87f9d923068 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/56ac2f2f78c844d8ad74b87f9d923068 2024-11-21T01:46:40,263 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/1641efd28c18412d908e4d51ef3d4f59 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/1641efd28c18412d908e4d51ef3d4f59 2024-11-21T01:46:40,264 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/f68223045ee54355a0366b7ab35174a0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/f68223045ee54355a0366b7ab35174a0 2024-11-21T01:46:40,265 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/5e4ed402d40a427aaad3b429c4d6daee to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/5e4ed402d40a427aaad3b429c4d6daee 2024-11-21T01:46:40,266 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/ded45e31221c40419d1d252b5c6b0dc2 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/ded45e31221c40419d1d252b5c6b0dc2 2024-11-21T01:46:40,266 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/3702d53c68ce4dc0be534f1c0b376447 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/3702d53c68ce4dc0be534f1c0b376447 2024-11-21T01:46:40,267 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/3a87aed54d794522ad0ad9c264a497f3 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/3a87aed54d794522ad0ad9c264a497f3 2024-11-21T01:46:40,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/3c566c9635f6428f959a8bf8bdc738d4 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/3c566c9635f6428f959a8bf8bdc738d4 2024-11-21T01:46:40,269 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/2e19af4ece174fe6bd61ca8cbbde2a76 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/2e19af4ece174fe6bd61ca8cbbde2a76 2024-11-21T01:46:40,270 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/cb32703c3d8c4002890be12009ea8edc to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/cb32703c3d8c4002890be12009ea8edc 2024-11-21T01:46:40,270 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/dafef24ba08243e888abdec6c89d1357 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/dafef24ba08243e888abdec6c89d1357 2024-11-21T01:46:40,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/ea3d9c1d94c74620b2ca662e0fb0c3e8, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/106a718c347a4aa49bcc28469e28c383, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/f4737e475a6649c69824655e4accb6f5, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/7b97a41ace234bfca5ed75599b6fda82, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/0bba499e048f41d7885bca7626177462, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/34e19aec69894be687734e9e46e0d6e0, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/ef66995a386e480ea8425ac8eb1d6ac8, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/374fa21782d24f4bb3f69e59cb5e1366, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/3f6313003d514f5f87283d694c787f75, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/f5d772f4f76442eca52b01a699cb3477, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/c01ec94b780a4022b1c066f5a708f79e, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/ee2733f9297d488cbbee3201fdf51447, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/0eeb9c48a73b44659f15451fb8d8269c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/f8ada7251a224ad3b45add5b6b9ac307, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/f48591b9efe743db8bb3057b7c5f0141, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/c7a15f8d2a174a7897e680a35349a6b4, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/c9afeb9a2abd4216adfcbca78b01ebf0, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/723a907395324090bcd71bb1e7ffb608, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/dfb37d1712b84f378505ca723a3c15a3, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/8a4f0c62e926461ebbc254b7b28ba9e6, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/556131495b2a454696c8187101081fe3, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/d3faa3f2391f4f428e7d8cc786e41247, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/2f701a6a3b7e42e2b3758a29f4c2d58c, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/94634762b21048d2bb289bf50a20fe0e] to archive 2024-11-21T01:46:40,272 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T01:46:40,273 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/ea3d9c1d94c74620b2ca662e0fb0c3e8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/ea3d9c1d94c74620b2ca662e0fb0c3e8 2024-11-21T01:46:40,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/106a718c347a4aa49bcc28469e28c383 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/106a718c347a4aa49bcc28469e28c383 2024-11-21T01:46:40,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/f4737e475a6649c69824655e4accb6f5 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/f4737e475a6649c69824655e4accb6f5 2024-11-21T01:46:40,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/7b97a41ace234bfca5ed75599b6fda82 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/7b97a41ace234bfca5ed75599b6fda82 2024-11-21T01:46:40,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/0bba499e048f41d7885bca7626177462 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/0bba499e048f41d7885bca7626177462 2024-11-21T01:46:40,277 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/34e19aec69894be687734e9e46e0d6e0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/34e19aec69894be687734e9e46e0d6e0 2024-11-21T01:46:40,278 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/ef66995a386e480ea8425ac8eb1d6ac8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/ef66995a386e480ea8425ac8eb1d6ac8 2024-11-21T01:46:40,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/374fa21782d24f4bb3f69e59cb5e1366 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/374fa21782d24f4bb3f69e59cb5e1366 2024-11-21T01:46:40,280 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/3f6313003d514f5f87283d694c787f75 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/3f6313003d514f5f87283d694c787f75 2024-11-21T01:46:40,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/f5d772f4f76442eca52b01a699cb3477 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/f5d772f4f76442eca52b01a699cb3477 2024-11-21T01:46:40,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/c01ec94b780a4022b1c066f5a708f79e to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/c01ec94b780a4022b1c066f5a708f79e 2024-11-21T01:46:40,283 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/ee2733f9297d488cbbee3201fdf51447 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/ee2733f9297d488cbbee3201fdf51447 2024-11-21T01:46:40,283 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/0eeb9c48a73b44659f15451fb8d8269c to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/0eeb9c48a73b44659f15451fb8d8269c 2024-11-21T01:46:40,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/f8ada7251a224ad3b45add5b6b9ac307 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/f8ada7251a224ad3b45add5b6b9ac307 2024-11-21T01:46:40,285 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/f48591b9efe743db8bb3057b7c5f0141 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/f48591b9efe743db8bb3057b7c5f0141 2024-11-21T01:46:40,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/c7a15f8d2a174a7897e680a35349a6b4 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/c7a15f8d2a174a7897e680a35349a6b4 2024-11-21T01:46:40,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/c9afeb9a2abd4216adfcbca78b01ebf0 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/c9afeb9a2abd4216adfcbca78b01ebf0 2024-11-21T01:46:40,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/723a907395324090bcd71bb1e7ffb608 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/723a907395324090bcd71bb1e7ffb608 2024-11-21T01:46:40,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/dfb37d1712b84f378505ca723a3c15a3 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/dfb37d1712b84f378505ca723a3c15a3 2024-11-21T01:46:40,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/8a4f0c62e926461ebbc254b7b28ba9e6 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/8a4f0c62e926461ebbc254b7b28ba9e6 2024-11-21T01:46:40,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/556131495b2a454696c8187101081fe3 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/556131495b2a454696c8187101081fe3 2024-11-21T01:46:40,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/d3faa3f2391f4f428e7d8cc786e41247 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/d3faa3f2391f4f428e7d8cc786e41247 2024-11-21T01:46:40,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/2f701a6a3b7e42e2b3758a29f4c2d58c to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/2f701a6a3b7e42e2b3758a29f4c2d58c 2024-11-21T01:46:40,293 DEBUG [StoreCloser-TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/94634762b21048d2bb289bf50a20fe0e to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/94634762b21048d2bb289bf50a20fe0e 2024-11-21T01:46:40,296 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/recovered.edits/430.seqid, newMaxSeqId=430, maxSeqId=4 2024-11-21T01:46:40,297 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. 2024-11-21T01:46:40,297 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(1676): Region close journal for f1185a7567ce5119c4e675690fbe9d2b: Waiting for close lock at 1732153598578Running coprocessor pre-close hooks at 1732153598578Disabling compacts and flushes for region at 1732153598578Disabling writes for close at 1732153598578Obtaining lock to block concurrent updates at 1732153598578Preparing flush snapshotting stores in f1185a7567ce5119c4e675690fbe9d2b at 1732153598578Finished memstore snapshotting TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732153598579 (+1 ms)Flushing stores of TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b. at 1732153598579Flushing f1185a7567ce5119c4e675690fbe9d2b/A: creating writer at 1732153598579Flushing mob file f1185a7567ce5119c4e675690fbe9d2b/A: appending metadata at 1732153598583 (+4 ms)Flushing mob file f1185a7567ce5119c4e675690fbe9d2b/A: closing flushed file at 1732153598583Flushing f1185a7567ce5119c4e675690fbe9d2b/A: appending metadata at 1732153598990 (+407 ms)Flushing f1185a7567ce5119c4e675690fbe9d2b/A: closing flushed file at 1732153598991 (+1 ms)Flushing f1185a7567ce5119c4e675690fbe9d2b/B: creating writer at 1732153599398 (+407 ms)Flushing f1185a7567ce5119c4e675690fbe9d2b/B: appending metadata at 1732153599400 (+2 ms)Flushing f1185a7567ce5119c4e675690fbe9d2b/B: closing flushed file at 1732153599400Flushing f1185a7567ce5119c4e675690fbe9d2b/C: creating writer at 1732153599807 (+407 ms)Flushing f1185a7567ce5119c4e675690fbe9d2b/C: appending metadata at 1732153599809 (+2 ms)Flushing f1185a7567ce5119c4e675690fbe9d2b/C: closing flushed file at 1732153599809Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42f29393: reopening flushed file at 1732153600216 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9e2c37e: reopening flushed file at 1732153600220 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@436adffe: reopening flushed file at 1732153600224 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f1185a7567ce5119c4e675690fbe9d2b in 1649ms, sequenceid=427, compaction requested=true at 1732153600228 (+4 ms)Writing region close event to WAL at 1732153600294 (+66 ms)Running coprocessor post-close hooks at 1732153600297 (+3 ms)Closed at 1732153600297 2024-11-21T01:46:40,299 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] handler.UnassignRegionHandler(157): Closed f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:40,299 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=184 updating hbase:meta row=f1185a7567ce5119c4e675690fbe9d2b, regionState=CLOSED 2024-11-21T01:46:40,300 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=185, ppid=184, state=RUNNABLE, hasLock=false; CloseRegionProcedure f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 because future has completed 2024-11-21T01:46:40,302 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=185, resume processing ppid=184 2024-11-21T01:46:40,302 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=185, ppid=184, state=SUCCESS, hasLock=false; CloseRegionProcedure f1185a7567ce5119c4e675690fbe9d2b, server=b93ada835ef8,41129,1732153434359 in 1.8750 sec 2024-11-21T01:46:40,303 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=184, resume processing ppid=183 2024-11-21T01:46:40,303 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=184, ppid=183, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f1185a7567ce5119c4e675690fbe9d2b, UNASSIGN in 1.8790 sec 2024-11-21T01:46:40,305 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=183, resume processing ppid=182 2024-11-21T01:46:40,305 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=183, ppid=182, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8830 sec 2024-11-21T01:46:40,307 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732153600307"}]},"ts":"1732153600307"} 2024-11-21T01:46:40,308 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T01:46:40,308 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T01:46:40,310 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=182, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8920 sec 2024-11-21T01:46:40,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-21T01:46:40,563 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:46:40,564 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-21T01:46:40,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] procedure2.ProcedureExecutor(1139): Stored pid=186, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:46:40,566 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=186, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:46:40,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=186 2024-11-21T01:46:40,566 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=186, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:46:40,569 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:40,571 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A, FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B, FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C, FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/recovered.edits] 2024-11-21T01:46:40,573 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/14693c634d654681b008f1bdace2d6aa to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/14693c634d654681b008f1bdace2d6aa 2024-11-21T01:46:40,574 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/5c54706fe72844c5b8bcf3fc5d288609 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/5c54706fe72844c5b8bcf3fc5d288609 2024-11-21T01:46:40,575 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/9f2087028b9443cfb6a23cd52e0254e8 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/9f2087028b9443cfb6a23cd52e0254e8 2024-11-21T01:46:40,576 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/b1b42505396d4c569cc93a29a8b43b7e to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/A/b1b42505396d4c569cc93a29a8b43b7e 2024-11-21T01:46:40,578 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/0db9c77609e04df5b92cb4e2dfba2c22 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/0db9c77609e04df5b92cb4e2dfba2c22 2024-11-21T01:46:40,579 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/1567410e67704b589b1d94dedb746c0a to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/1567410e67704b589b1d94dedb746c0a 2024-11-21T01:46:40,579 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/2a101083cbd24d678349efd7a170445a to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/2a101083cbd24d678349efd7a170445a 2024-11-21T01:46:40,580 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/ab7891097c5d4136b1fa1536f95dcfa1 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/B/ab7891097c5d4136b1fa1536f95dcfa1 2024-11-21T01:46:40,582 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/049adb772c5c489db92c8f29512c4f62 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/049adb772c5c489db92c8f29512c4f62 2024-11-21T01:46:40,583 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/1b6e717383c64e99a4b6917a28421b2a to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/1b6e717383c64e99a4b6917a28421b2a 2024-11-21T01:46:40,584 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/9296a75f0e8542a9b3ea27725cef9c61 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/9296a75f0e8542a9b3ea27725cef9c61 2024-11-21T01:46:40,584 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/fdd5999231544e17acdbfb96cb9f0af3 to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/C/fdd5999231544e17acdbfb96cb9f0af3 2024-11-21T01:46:40,587 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/recovered.edits/430.seqid to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b/recovered.edits/430.seqid 2024-11-21T01:46:40,587 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/default/TestAcidGuarantees/f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:40,587 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T01:46:40,587 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T01:46:40,588 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-21T01:46:40,590 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112108665353da6746d5b3effac7546d7bd0_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112108665353da6746d5b3effac7546d7bd0_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:40,591 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213ed6341309a04bb8a54ad206d095b2d1_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213ed6341309a04bb8a54ad206d095b2d1_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:40,592 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214c0a645b67bf40c291aff843bb916dac_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214c0a645b67bf40c291aff843bb916dac_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:40,593 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215810ae39f36c43c5acc8ef2b27985c19_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215810ae39f36c43c5acc8ef2b27985c19_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:40,595 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112158e0556bd4a841e39f744f333364b244_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112158e0556bd4a841e39f744f333364b244_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:40,596 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216e0fdd4d81e3469fbd33757d0098e37a_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216e0fdd4d81e3469fbd33757d0098e37a_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:40,596 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217ad810caf4754566af3c59295d9f02b7_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217ad810caf4754566af3c59295d9f02b7_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:40,598 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217ecaacd968804c9fa1377ae3fa7bd7c4_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217ecaacd968804c9fa1377ae3fa7bd7c4_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:40,599 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a1d79a3e50bd47a38f6876aa77c77c46_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a1d79a3e50bd47a38f6876aa77c77c46_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:40,600 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a87f1b049850421a8d5c0ee7f0c93a80_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a87f1b049850421a8d5c0ee7f0c93a80_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:40,601 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c053839637a74214ab56c2aacdcddea6_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c053839637a74214ab56c2aacdcddea6_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:40,602 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cc0cac1762414e14b9b53cb413e6a9ed_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cc0cac1762414e14b9b53cb413e6a9ed_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:40,602 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d047de01238743689dfb505a9bd4771e_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d047de01238743689dfb505a9bd4771e_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:40,603 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d8d5928ed63944b08b7c005c6c97d3c2_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d8d5928ed63944b08b7c005c6c97d3c2_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:40,605 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121de06fd8c936d45a394193e77c98e6e58_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121de06fd8c936d45a394193e77c98e6e58_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:40,606 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121dfdf475b2ffe4aa7a9b9fa6e95a61dd5_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121dfdf475b2ffe4aa7a9b9fa6e95a61dd5_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:40,607 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ecc627d030cc426b8a4058f4798036cc_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ecc627d030cc426b8a4058f4798036cc_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:40,608 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ef713752a2824010a0a78160d596db50_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ef713752a2824010a0a78160d596db50_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:40,609 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f27aecd2005e424190257b80c8f65d91_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f27aecd2005e424190257b80c8f65d91_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:40,610 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f38e0016044c4d1c85f1d7a167a95b0f_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f38e0016044c4d1c85f1d7a167a95b0f_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:40,611 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ff578c66d19846d2bc414f050b9bb8dd_f1185a7567ce5119c4e675690fbe9d2b to hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ff578c66d19846d2bc414f050b9bb8dd_f1185a7567ce5119c4e675690fbe9d2b 2024-11-21T01:46:40,611 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T01:46:40,613 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=186, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:46:40,616 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T01:46:40,618 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T01:46:40,619 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=186, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:46:40,619 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T01:46:40,620 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732153600619"}]},"ts":"9223372036854775807"} 2024-11-21T01:46:40,621 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T01:46:40,621 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f1185a7567ce5119c4e675690fbe9d2b, NAME => 'TestAcidGuarantees,,1732153570938.f1185a7567ce5119c4e675690fbe9d2b.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T01:46:40,621 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T01:46:40,622 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732153600622"}]},"ts":"9223372036854775807"} 2024-11-21T01:46:40,623 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T01:46:40,624 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=186, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T01:46:40,625 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=186, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 60 msec 2024-11-21T01:46:40,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=186 2024-11-21T01:46:40,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T01:46:40,673 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T01:46:40,683 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251 (was 250) - Thread LEAK? -, OpenFileDescriptor=451 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=577 (was 619), ProcessCount=11 (was 11), AvailableMemoryMB=2189 (was 3322) 2024-11-21T01:46:40,683 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-21T01:46:40,683 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T01:46:40,683 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T01:46:40,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:40,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:40,683 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:46:40,683 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-21T01:46:40,683 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=974344401, stopped=false 2024-11-21T01:46:40,684 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=b93ada835ef8,36607,1732153433607 2024-11-21T01:46:40,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36607-0x10112f6cf300000, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T01:46:40,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36607-0x10112f6cf300000, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:46:40,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41129-0x10112f6cf300001, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T01:46:40,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41129-0x10112f6cf300001, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:46:40,685 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T01:46:40,686 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41129-0x10112f6cf300001, quorum=127.0.0.1:57444, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T01:46:40,686 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T01:46:40,686 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T01:46:40,687 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36607-0x10112f6cf300000, quorum=127.0.0.1:57444, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T01:46:40,687 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:40,687 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b93ada835ef8,41129,1732153434359' ***** 2024-11-21T01:46:40,687 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T01:46:40,687 INFO [RS:0;b93ada835ef8:41129 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T01:46:40,688 INFO [RS:0;b93ada835ef8:41129 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T01:46:40,688 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T01:46:40,688 INFO [RS:0;b93ada835ef8:41129 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T01:46:40,688 INFO [RS:0;b93ada835ef8:41129 {}] regionserver.HRegionServer(959): stopping server b93ada835ef8,41129,1732153434359 2024-11-21T01:46:40,688 INFO [RS:0;b93ada835ef8:41129 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T01:46:40,688 INFO [RS:0;b93ada835ef8:41129 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;b93ada835ef8:41129. 2024-11-21T01:46:40,688 DEBUG [RS:0;b93ada835ef8:41129 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T01:46:40,688 DEBUG [RS:0;b93ada835ef8:41129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:40,688 INFO [RS:0;b93ada835ef8:41129 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T01:46:40,688 INFO [RS:0;b93ada835ef8:41129 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T01:46:40,688 INFO [RS:0;b93ada835ef8:41129 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T01:46:40,689 INFO [RS:0;b93ada835ef8:41129 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-21T01:46:40,689 INFO [RS:0;b93ada835ef8:41129 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-21T01:46:40,689 DEBUG [RS:0;b93ada835ef8:41129 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-21T01:46:40,690 DEBUG [RS:0;b93ada835ef8:41129 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T01:46:40,690 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T01:46:40,690 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T01:46:40,690 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T01:46:40,690 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T01:46:40,690 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T01:46:40,690 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-21T01:46:40,715 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/hbase/meta/1588230740/.tmp/info/e436899a303f4a9a9d647b80c5bbb5ba is 95, key is TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc./info:/1732153464980/DeleteFamily/seqid=0 2024-11-21T01:46:40,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742504_1680 (size=6514) 2024-11-21T01:46:40,741 INFO [regionserver/b93ada835ef8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T01:46:40,890 DEBUG [RS:0;b93ada835ef8:41129 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T01:46:41,090 DEBUG [RS:0;b93ada835ef8:41129 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T01:46:41,118 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/hbase/meta/1588230740/.tmp/info/e436899a303f4a9a9d647b80c5bbb5ba 2024-11-21T01:46:41,137 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/hbase/meta/1588230740/.tmp/ns/c736750f790e46c9ad02b464849632b7 is 93, key is TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc./ns:/1732153464939/DeleteFamily/seqid=0 2024-11-21T01:46:41,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742505_1681 (size=5906) 2024-11-21T01:46:41,290 DEBUG [RS:0;b93ada835ef8:41129 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T01:46:41,490 DEBUG [RS:0;b93ada835ef8:41129 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T01:46:41,540 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/hbase/meta/1588230740/.tmp/ns/c736750f790e46c9ad02b464849632b7 2024-11-21T01:46:41,560 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/hbase/meta/1588230740/.tmp/rep_barrier/895d8b55f38e4a66a0b1c11d273ad46b is 102, key is TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc./rep_barrier:/1732153464939/DeleteFamily/seqid=0 2024-11-21T01:46:41,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742506_1682 (size=6025) 2024-11-21T01:46:41,657 INFO [regionserver/b93ada835ef8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-21T01:46:41,657 INFO [regionserver/b93ada835ef8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-21T01:46:41,691 DEBUG [RS:0;b93ada835ef8:41129 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T01:46:41,891 DEBUG [RS:0;b93ada835ef8:41129 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T01:46:41,964 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/hbase/meta/1588230740/.tmp/rep_barrier/895d8b55f38e4a66a0b1c11d273ad46b 2024-11-21T01:46:41,983 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/hbase/meta/1588230740/.tmp/table/58ba2ec517bc466f83686b0d06bcfe52 is 96, key is TestAcidGuarantees,,1732153438208.06db9be3c4e6682aa8966442e1ef70cc./table:/1732153464939/DeleteFamily/seqid=0 2024-11-21T01:46:41,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742507_1683 (size=5936) 2024-11-21T01:46:42,091 DEBUG [RS:0;b93ada835ef8:41129 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T01:46:42,291 DEBUG [RS:0;b93ada835ef8:41129 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T01:46:42,386 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/hbase/meta/1588230740/.tmp/table/58ba2ec517bc466f83686b0d06bcfe52 2024-11-21T01:46:42,389 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/hbase/meta/1588230740/.tmp/info/e436899a303f4a9a9d647b80c5bbb5ba as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/hbase/meta/1588230740/info/e436899a303f4a9a9d647b80c5bbb5ba 2024-11-21T01:46:42,392 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/hbase/meta/1588230740/info/e436899a303f4a9a9d647b80c5bbb5ba, entries=12, sequenceid=90, filesize=6.4 K 2024-11-21T01:46:42,393 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/hbase/meta/1588230740/.tmp/ns/c736750f790e46c9ad02b464849632b7 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/hbase/meta/1588230740/ns/c736750f790e46c9ad02b464849632b7 2024-11-21T01:46:42,396 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/hbase/meta/1588230740/ns/c736750f790e46c9ad02b464849632b7, entries=8, sequenceid=90, filesize=5.8 K 2024-11-21T01:46:42,396 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/hbase/meta/1588230740/.tmp/rep_barrier/895d8b55f38e4a66a0b1c11d273ad46b as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/hbase/meta/1588230740/rep_barrier/895d8b55f38e4a66a0b1c11d273ad46b 2024-11-21T01:46:42,399 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/hbase/meta/1588230740/rep_barrier/895d8b55f38e4a66a0b1c11d273ad46b, entries=6, sequenceid=90, filesize=5.9 K 2024-11-21T01:46:42,400 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/hbase/meta/1588230740/.tmp/table/58ba2ec517bc466f83686b0d06bcfe52 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/hbase/meta/1588230740/table/58ba2ec517bc466f83686b0d06bcfe52 2024-11-21T01:46:42,402 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/hbase/meta/1588230740/table/58ba2ec517bc466f83686b0d06bcfe52, entries=7, sequenceid=90, filesize=5.8 K 2024-11-21T01:46:42,403 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1713ms, sequenceid=90, compaction requested=false 2024-11-21T01:46:42,406 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-21T01:46:42,407 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T01:46:42,407 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T01:46:42,407 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732153600689Running coprocessor pre-close hooks at 1732153600690 (+1 ms)Disabling compacts and flushes for region at 1732153600690Disabling writes for close at 1732153600690Obtaining lock to block concurrent updates at 1732153600690Preparing flush snapshotting stores in 1588230740 at 1732153600690Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732153600690Flushing stores of hbase:meta,,1.1588230740 at 1732153600691 (+1 ms)Flushing 1588230740/info: creating writer at 1732153600691Flushing 1588230740/info: appending metadata at 1732153600714 (+23 ms)Flushing 1588230740/info: closing flushed file at 1732153600714Flushing 1588230740/ns: creating writer at 1732153601122 (+408 ms)Flushing 1588230740/ns: appending metadata at 1732153601136 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732153601136Flushing 1588230740/rep_barrier: creating writer at 1732153601543 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732153601560 (+17 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732153601560Flushing 1588230740/table: creating writer at 1732153601968 (+408 ms)Flushing 1588230740/table: appending metadata at 1732153601982 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732153601982Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@abaf5bb: reopening flushed file at 1732153602389 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27147d94: reopening flushed file at 1732153602393 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@126f8d60: reopening flushed file at 1732153602396 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a8c98cc: reopening flushed file at 1732153602399 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1713ms, sequenceid=90, compaction requested=false at 1732153602403 (+4 ms)Writing region close event to WAL at 1732153602404 (+1 ms)Running coprocessor post-close hooks at 1732153602406 (+2 ms)Closed at 1732153602407 (+1 ms) 2024-11-21T01:46:42,407 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-21T01:46:42,492 INFO [RS:0;b93ada835ef8:41129 {}] regionserver.HRegionServer(976): stopping server b93ada835ef8,41129,1732153434359; all regions closed. 2024-11-21T01:46:42,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741834_1010 (size=25344) 2024-11-21T01:46:42,498 DEBUG [RS:0;b93ada835ef8:41129 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/oldWALs 2024-11-21T01:46:42,498 INFO [RS:0;b93ada835ef8:41129 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b93ada835ef8%2C41129%2C1732153434359.meta:.meta(num 1732153437723) 2024-11-21T01:46:42,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741832_1008 (size=17958844) 2024-11-21T01:46:42,501 DEBUG [RS:0;b93ada835ef8:41129 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/oldWALs 2024-11-21T01:46:42,501 INFO [RS:0;b93ada835ef8:41129 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b93ada835ef8%2C41129%2C1732153434359:(num 1732153436809) 2024-11-21T01:46:42,501 DEBUG [RS:0;b93ada835ef8:41129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:46:42,501 INFO [RS:0;b93ada835ef8:41129 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T01:46:42,502 INFO [RS:0;b93ada835ef8:41129 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T01:46:42,502 INFO [RS:0;b93ada835ef8:41129 {}] hbase.ChoreService(370): Chore service for: regionserver/b93ada835ef8:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-21T01:46:42,502 INFO [regionserver/b93ada835ef8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T01:46:42,502 INFO [RS:0;b93ada835ef8:41129 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T01:46:42,503 INFO [RS:0;b93ada835ef8:41129 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41129 2024-11-21T01:46:42,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41129-0x10112f6cf300001, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b93ada835ef8,41129,1732153434359 2024-11-21T01:46:42,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36607-0x10112f6cf300000, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T01:46:42,507 INFO [RS:0;b93ada835ef8:41129 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T01:46:42,509 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b93ada835ef8,41129,1732153434359] 2024-11-21T01:46:42,510 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b93ada835ef8,41129,1732153434359 already deleted, retry=false 2024-11-21T01:46:42,510 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; b93ada835ef8,41129,1732153434359 expired; onlineServers=0 2024-11-21T01:46:42,510 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'b93ada835ef8,36607,1732153433607' ***** 2024-11-21T01:46:42,510 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-21T01:46:42,511 INFO [M:0;b93ada835ef8:36607 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T01:46:42,511 INFO [M:0;b93ada835ef8:36607 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T01:46:42,511 DEBUG [M:0;b93ada835ef8:36607 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-21T01:46:42,511 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-21T01:46:42,511 DEBUG [M:0;b93ada835ef8:36607 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-21T01:46:42,511 DEBUG [master/b93ada835ef8:0:becomeActiveMaster-HFileCleaner.small.0-1732153436445 {}] cleaner.HFileCleaner(306): Exit Thread[master/b93ada835ef8:0:becomeActiveMaster-HFileCleaner.small.0-1732153436445,5,FailOnTimeoutGroup] 2024-11-21T01:46:42,511 DEBUG [master/b93ada835ef8:0:becomeActiveMaster-HFileCleaner.large.0-1732153436444 {}] cleaner.HFileCleaner(306): Exit Thread[master/b93ada835ef8:0:becomeActiveMaster-HFileCleaner.large.0-1732153436444,5,FailOnTimeoutGroup] 2024-11-21T01:46:42,511 INFO [M:0;b93ada835ef8:36607 {}] hbase.ChoreService(370): Chore service for: master/b93ada835ef8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-21T01:46:42,511 INFO [M:0;b93ada835ef8:36607 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T01:46:42,511 DEBUG [M:0;b93ada835ef8:36607 {}] master.HMaster(1795): Stopping service threads 2024-11-21T01:46:42,511 INFO [M:0;b93ada835ef8:36607 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-21T01:46:42,511 INFO [M:0;b93ada835ef8:36607 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T01:46:42,512 ERROR [M:0;b93ada835ef8:36607 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:37693,5,PEWorkerGroup] Thread[IPC Client (1297744652) connection to localhost/127.0.0.1:37693 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:37693,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-21T01:46:42,512 INFO [M:0;b93ada835ef8:36607 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T01:46:42,512 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-21T01:46:42,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36607-0x10112f6cf300000, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-21T01:46:42,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36607-0x10112f6cf300000, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:46:42,513 DEBUG [M:0;b93ada835ef8:36607 {}] zookeeper.ZKUtil(347): master:36607-0x10112f6cf300000, quorum=127.0.0.1:57444, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-21T01:46:42,513 WARN [M:0;b93ada835ef8:36607 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-21T01:46:42,513 INFO [M:0;b93ada835ef8:36607 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/.lastflushedseqids 2024-11-21T01:46:42,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742508_1684 (size=72) 2024-11-21T01:46:42,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41129-0x10112f6cf300001, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T01:46:42,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41129-0x10112f6cf300001, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T01:46:42,610 INFO [RS:0;b93ada835ef8:41129 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T01:46:42,610 INFO [RS:0;b93ada835ef8:41129 {}] regionserver.HRegionServer(1031): Exiting; stopping=b93ada835ef8,41129,1732153434359; zookeeper connection closed. 2024-11-21T01:46:42,611 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@9c35628 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@9c35628 2024-11-21T01:46:42,611 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-21T01:46:42,923 INFO [M:0;b93ada835ef8:36607 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T01:46:42,923 INFO [M:0;b93ada835ef8:36607 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-21T01:46:42,923 DEBUG [M:0;b93ada835ef8:36607 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T01:46:42,923 INFO [M:0;b93ada835ef8:36607 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T01:46:42,923 DEBUG [M:0;b93ada835ef8:36607 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T01:46:42,923 DEBUG [M:0;b93ada835ef8:36607 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T01:46:42,923 DEBUG [M:0;b93ada835ef8:36607 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T01:46:42,923 INFO [M:0;b93ada835ef8:36607 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=802.32 KB heapSize=984.57 KB 2024-11-21T01:46:42,939 DEBUG [M:0;b93ada835ef8:36607 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d071ec49cd894bcfa0bbd437676ed223 is 82, key is hbase:meta,,1/info:regioninfo/1732153437841/Put/seqid=0 2024-11-21T01:46:42,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742509_1685 (size=5672) 2024-11-21T01:46:43,343 INFO [M:0;b93ada835ef8:36607 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2245 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d071ec49cd894bcfa0bbd437676ed223 2024-11-21T01:46:43,370 DEBUG [M:0;b93ada835ef8:36607 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c31220b0a32d48e291bb23c21ac1a13d is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x9D/proc:d/1732153573620/Put/seqid=0 2024-11-21T01:46:43,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742510_1686 (size=48148) 2024-11-21T01:46:43,775 INFO [M:0;b93ada835ef8:36607 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=801.76 KB at sequenceid=2245 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c31220b0a32d48e291bb23c21ac1a13d 2024-11-21T01:46:43,778 INFO [M:0;b93ada835ef8:36607 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c31220b0a32d48e291bb23c21ac1a13d 2024-11-21T01:46:43,794 DEBUG [M:0;b93ada835ef8:36607 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/72b9034aff354ac3824e58aeca515c84 is 69, key is b93ada835ef8,41129,1732153434359/rs:state/1732153436551/Put/seqid=0 2024-11-21T01:46:43,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073742511_1687 (size=5156) 2024-11-21T01:46:44,052 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T01:46:44,052 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-21T01:46:44,053 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-21T01:46:44,198 INFO [M:0;b93ada835ef8:36607 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2245 (bloomFilter=true), to=hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/72b9034aff354ac3824e58aeca515c84 2024-11-21T01:46:44,202 DEBUG [M:0;b93ada835ef8:36607 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d071ec49cd894bcfa0bbd437676ed223 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d071ec49cd894bcfa0bbd437676ed223 2024-11-21T01:46:44,205 INFO [M:0;b93ada835ef8:36607 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d071ec49cd894bcfa0bbd437676ed223, entries=8, sequenceid=2245, filesize=5.5 K 2024-11-21T01:46:44,206 DEBUG [M:0;b93ada835ef8:36607 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c31220b0a32d48e291bb23c21ac1a13d as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c31220b0a32d48e291bb23c21ac1a13d 2024-11-21T01:46:44,209 INFO [M:0;b93ada835ef8:36607 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c31220b0a32d48e291bb23c21ac1a13d 2024-11-21T01:46:44,209 INFO [M:0;b93ada835ef8:36607 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c31220b0a32d48e291bb23c21ac1a13d, entries=186, sequenceid=2245, filesize=47.0 K 2024-11-21T01:46:44,217 DEBUG [M:0;b93ada835ef8:36607 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/72b9034aff354ac3824e58aeca515c84 as hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/72b9034aff354ac3824e58aeca515c84 2024-11-21T01:46:44,221 INFO [M:0;b93ada835ef8:36607 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37693/user/jenkins/test-data/623fcd1f-c32a-c710-9fcf-d419f2ab7f1d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/72b9034aff354ac3824e58aeca515c84, entries=1, sequenceid=2245, filesize=5.0 K 2024-11-21T01:46:44,222 INFO [M:0;b93ada835ef8:36607 {}] regionserver.HRegion(3140): Finished flush of dataSize ~802.32 KB/821573, heapSize ~984.27 KB/1007896, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1299ms, sequenceid=2245, compaction requested=false 2024-11-21T01:46:44,230 INFO [M:0;b93ada835ef8:36607 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T01:46:44,230 DEBUG [M:0;b93ada835ef8:36607 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732153602923Disabling compacts and flushes for region at 1732153602923Disabling writes for close at 1732153602923Obtaining lock to block concurrent updates at 1732153602923Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732153602923Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=821573, getHeapSize=1008136, getOffHeapSize=0, getCellsCount=2456 at 1732153602924 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732153602924Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732153602924Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732153602939 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732153602939Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732153603347 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732153603370 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732153603370Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732153603779 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732153603794 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732153603794Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b8ade81: reopening flushed file at 1732153604202 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1821c6c8: reopening flushed file at 1732153604205 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43a5d7bd: reopening flushed file at 1732153604209 (+4 ms)Finished flush of dataSize ~802.32 KB/821573, heapSize ~984.27 KB/1007896, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1299ms, sequenceid=2245, compaction requested=false at 1732153604222 (+13 ms)Writing region close event to WAL at 1732153604230 (+8 ms)Closed at 1732153604230 2024-11-21T01:46:44,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45349 is added to blk_1073741830_1006 (size=968761) 2024-11-21T01:46:44,235 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T01:46:44,235 INFO [M:0;b93ada835ef8:36607 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-21T01:46:44,235 INFO [M:0;b93ada835ef8:36607 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36607 2024-11-21T01:46:44,235 INFO [M:0;b93ada835ef8:36607 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T01:46:44,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36607-0x10112f6cf300000, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T01:46:44,338 INFO [M:0;b93ada835ef8:36607 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T01:46:44,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36607-0x10112f6cf300000, quorum=127.0.0.1:57444, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T01:46:44,344 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@72835e42{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T01:46:44,347 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2098bdc7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-21T01:46:44,347 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-21T01:46:44,347 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@54a7b62e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-21T01:46:44,347 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4828e07b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/hadoop.log.dir/,STOPPED} 2024-11-21T01:46:44,350 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-21T01:46:44,350 WARN [BP-1273700343-172.17.0.3-1732153430280 heartbeating to localhost/127.0.0.1:37693 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-21T01:46:44,350 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-21T01:46:44,350 WARN [BP-1273700343-172.17.0.3-1732153430280 heartbeating to localhost/127.0.0.1:37693 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1273700343-172.17.0.3-1732153430280 (Datanode Uuid 1349e6fd-a662-4842-aff7-117c872bca27) service to localhost/127.0.0.1:37693 2024-11-21T01:46:44,352 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/cluster_5ec1f663-147e-5567-4614-f2cd450fd939/data/data1/current/BP-1273700343-172.17.0.3-1732153430280 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-21T01:46:44,352 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/cluster_5ec1f663-147e-5567-4614-f2cd450fd939/data/data2/current/BP-1273700343-172.17.0.3-1732153430280 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-21T01:46:44,353 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-21T01:46:44,360 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2341a969{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-21T01:46:44,360 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-21T01:46:44,360 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-21T01:46:44,360 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-21T01:46:44,361 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/929b1af0-8ff5-5935-38dc-e3e155de26d0/hadoop.log.dir/,STOPPED} 2024-11-21T01:46:44,377 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-21T01:46:44,511 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down